OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: character handling fix (#2453)
Mike Fortman3eca82b1cfad
diff --git a/custom_components/openrag/opensearch_multimodal.py b/custom_components/openrag/opensearch_multimodal.pyindex ec4011dab..4a1b1c6eb 100644--- a/custom_components/openrag/opensearch_multimodal.py+++ b/custom_components/openrag/opensearch_multimodal.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport copyimport json+import unicodedataimport uuidfrom concurrent.futures import ThreadPoolExecutor, as_completedfrom dataclasses import dataclass@@ -33,6 +34,17 @@ SELECTED_EMBEDDING_PROVIDER_VAR = "SELECTED_EMBEDDING_PROVIDER"LEGACY_EMBEDDING_ROUTE_PREFIX = "legacy:"INDEXED_EMBEDDING_ROUTE_PREFIX = "space:"+# Some PDF fonts map common kanji (日, 月, 木, 金, 土, ...) to the look-alike+# Kangxi Radical code points in their /ToUnicode CMap, and Docling copies them+# through unchanged. Map just those blocks back to CJK Unified Ideographs;+# full NFKC would also rewrite full-width digits, ㈱, ①, half-width katakana.+# Mirrors normalize_cjk_radicals in src/utils/document_processing.py.+CJK_RADICAL_TABLE = {+ cp: unicodedata.normalize("NFKC", chr(cp))+ for cp in range(0x2E80, 0x2FE0) # CJK Radicals Supplement + Kangxi Radicals+ if unicodedata.normalize("NFKC", chr(cp)) != chr(cp)+}+@dataclass(frozen=True)class EmbeddingSpace:@@ -1537,6 +1549,8 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponfor doc_obj in docs:data_copy = json.loads(doc_obj.model_dump_json())text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)+ if isinstance(text, str):+ text = text.translate(CJK_RADICAL_TABLE)texts.append(text)# Merge additional metadata from table inputdiff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.jsonindex 64e97d696..357649a41 100644--- a/flows/ingestion_flow.json+++ b/flows/ingestion_flow.json@@ -4248,7 +4248,7 @@"last_updated": "2026-08-18T22:46:55.693Z","legacy": false,"metadata": {- "code_hash": "a3727d7c66eb",+ "code_hash": "1a2a5b8d0ca7","dependencies": {"dependencies": [{@@ -4409,7 +4409,7 @@"show": true,"title_case": false,"type": "code",- "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom dataclasses import dataclass\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\nEMBEDDING_SPACE_PAGE_SIZE = 100\nSELECTED_EMBEDDING_PROVIDER_VAR = \"SELECTED_EMBEDDING_PROVIDER\"\nLEGACY_EMBEDDING_ROUTE_PREFIX = \"legacy:\"\nINDEXED_EMBEDDING_ROUTE_PREFIX = \"space:\"\n\n\n@dataclass(frozen=True)\nclass EmbeddingSpace:\n \"\"\"One indexed vector space and the model route needed to query it.\"\"\"\n\n space_id: str\n route_model: str\n field_identity: str\n legacy: bool = False\n\n\ndef build_embedding_space_aggregation(\n *,\n size: int,\n qualified_after: dict[str, Any] | None = None,\n legacy_after: dict[str, Any] | None = None,\n include_qualified: bool = True,\n include_legacy: bool = True,\n) -> dict[str, Any]:\n \"\"\"Build pageable discovery aggregations for exact and legacy vector spaces.\"\"\"\n aggregations: dict[str, Any] = {}\n if include_qualified:\n composite: dict[str, Any] = {\n \"size\": size,\n \"sources\": [{\"space_id\": {\"terms\": {\"field\": \"embedding_space_id\"}}}],\n }\n if qualified_after:\n composite[\"after\"] = qualified_after\n aggregations[\"embedding_spaces\"] = {\"composite\": composite}\n\n if include_legacy:\n composite = {\n \"size\": size,\n \"sources\": [{\"model\": {\"terms\": {\"field\": \"embedding_model\"}}}],\n }\n if legacy_after:\n composite[\"after\"] = legacy_after\n aggregations[\"legacy_embedding_models\"] = {\"composite\": composite}\n return aggregations\n\n\ndef embedding_space_after_keys(\n result: dict[str, Any],\n) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:\n \"\"\"Return composite pagination cursors for exact and legacy aggregations.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_after = aggregations.get(\"embedding_spaces\", {}).get(\"after_key\")\n legacy_after = aggregations.get(\"legacy_embedding_models\", {}).get(\"after_key\")\n return qualified_after, legacy_after\n\n\ndef embedding_spaces_from_aggregation(result: dict[str, Any]) -> list[EmbeddingSpace]:\n \"\"\"Convert an OpenSearch aggregation response into stable retrieval identities.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_buckets = aggregations.get(\"embedding_spaces\", {}).get(\"buckets\", [])\n legacy_buckets = aggregations.get(\"legacy_embedding_models\", {}).get(\"buckets\", [])\n\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n for bucket in qualified_buckets:\n key = bucket.get(\"key\")\n raw_space_id = key.get(\"space_id\") if isinstance(key, dict) else key\n space_id = str(raw_space_id or \"\").strip()\n if not space_id or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=f\"{INDEXED_EMBEDDING_ROUTE_PREFIX}{space_id}\",\n field_identity=space_id,\n )\n )\n\n for bucket in legacy_buckets:\n key = bucket.get(\"key\")\n raw_model = key.get(\"model\") if isinstance(key, dict) else key\n model = str(raw_model or \"\").strip()\n space_id = f\"{LEGACY_EMBEDDING_ROUTE_PREFIX}{model}\"\n if not model or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=space_id,\n field_identity=model,\n legacy=True,\n )\n )\n return spaces\n\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {\n h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None\n }\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\nUNTRUSTED_CHUNK_FENCE_START = \"<<<UNTRUSTED_DOC_CHUNK>>>\"\nUNTRUSTED_CHUNK_FENCE_END = \"<<<END_UNTRUSTED_DOC_CHUNK>>>\"\n\n\ndef fence_untrusted_text(text: str) -> str:\n \"\"\"Wrap retrieved document text in delimiters marking it as untrusted data.\n\n Retrieved chunk text can contain attacker-controlled natural-language\n instructions (indirect prompt injection, VULN-13906). Fencing it lets the\n system prompt tell the LLM to treat anything between these markers as\n data only, never as instructions or tool directives.\n\n Any literal fence markers already present in `text` are escaped first, so\n a poisoned chunk can't embed a fake end-of-fence marker to break out of\n the untrusted section and have its continuation misread as trusted.\n \"\"\"\n if not text:\n return text\n escaped = text.replace(UNTRUSTED_CHUNK_FENCE_START, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_START).replace(\n UNTRUSTED_CHUNK_FENCE_END, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_END\n )\n return f\"{UNTRUSTED_CHUNK_FENCE_START}\\n{escaped}\\n{UNTRUSTED_CHUNK_FENCE_END}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"embedding_provider_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"embedding_provider_name\",\n display_name=\"Embedding Provider\",\n value=SELECTED_EMBEDDING_PROVIDER_VAR,\n info=(\n \"Exact OpenRAG provider key used for new ingestion metadata and vector fields. \"\n \"Bound to SELECTED_EMBEDDING_PROVIDER in managed OpenRAG flows.\"\n ),\n load_from_db=True,\n advanced=True,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"OPENSEARCH_PASSWORD\",\n load_from_db=True,\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False,\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n SecretStrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n if \"text\" in source:\n source[\"text\"] = fence_untrusted_text(source[\"text\"])\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n\n logger.debug(\n \"[OpenRAG callback config] configured for ingest_run_id=%s (url_len=%d, token_len=%d)\",\n ingest_run_id,\n len(url),\n len(token),\n )\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n logger.debug(\n f\"[OpenRAG ingest POST] total_chunks={len(requests)} batch_size={batch_size} timeout_s={timeout}\"\n )\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.debug(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n logger.debug(\n f\"[OpenRAG ingest POST resp] batch={batch_number} status={response.status_code}\"\n )\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code})\"\n )\n logger.warning(msg)\n raise RuntimeError(msg)\n\n logger.debug(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n embedding_provider: str | None = None,\n embedding_space_id: str | None = None,\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n embedding_provider: Exact provider key, when known\n embedding_space_id: Provider-qualified vector-space identity\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if embedding_provider:\n request[\"embedding_provider\"] = embedding_provider\n if embedding_space_id:\n request[\"embedding_space_id\"] = embedding_space_id\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", url),\n (\"openrag_ingest_token\", token),\n (\"openrag_ingest_run_id\", ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self._openrag_input_to_str(self.password) or \"\").strip()\n if pwd == \"OPENSEARCH_PASSWORD\" or not pwd:\n from config.settings import get_opensearch_password\n\n pwd = get_opensearch_password() or \"\"\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n embedding_provider = self._openrag_input_to_str(\n getattr(self, \"embedding_provider_name\", \"\")\n ).lower()\n if embedding_provider == SELECTED_EMBEDDING_PROVIDER_VAR.lower():\n embedding_provider = \"\"\n embedding_space_id = (\n f\"{embedding_provider}:{embedding_model}\" if embedding_provider else None\n )\n dynamic_field_name = get_embedding_field_name(embedding_space_id or embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n embedding_provider=embedding_provider or None,\n embedding_space_id=embedding_space_id,\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(\n self, client: OpenSearch, filter_clauses: list[dict] | None = None\n ) -> list[EmbeddingSpace]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n result: dict[str, Any] = {}\n\n # Composite aggregations must be top-level. Exact and legacy\n # spaces use different document filters, so paginate them in two\n # independent passes rather than nesting composite under filter.\n for legacy in (False, True):\n after = None\n while True:\n agg_query = {\n \"size\": 0,\n \"aggs\": build_embedding_space_aggregation(\n size=EMBEDDING_SPACE_PAGE_SIZE,\n qualified_after=None if legacy else after,\n legacy_after=after if legacy else None,\n include_qualified=not legacy,\n include_legacy=legacy,\n ),\n }\n bool_query: dict[str, Any] = {}\n if filter_clauses:\n bool_query[\"filter\"] = filter_clauses\n if legacy:\n bool_query[\"must_not\"] = [{\"exists\": {\"field\": \"embedding_space_id\"}}]\n if bool_query:\n agg_query[\"query\"] = {\"bool\": bool_query}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n for space in embedding_spaces_from_aggregation(result):\n if space.space_id not in seen:\n seen.add(space.space_id)\n spaces.append(space)\n\n qualified_after, legacy_after = embedding_space_after_keys(result)\n next_after = legacy_after if legacy else qualified_after\n if not next_after or next_after == after:\n break\n after = next_after\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding spaces in corpus: {[space.space_id for space in spaces]}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not spaces:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Provider provenance cannot be inferred safely after discovery\n # fails. Keyword search remains available without guessing.\n return []\n else:\n return spaces\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_spaces = self._detect_available_models(client, filter_clauses)\n\n if not available_spaces:\n logger.warning(\"No embedding spaces found in index; using keyword-only search\")\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.warning(\n \"No valid embedding adapter is available; continuing with keyword-only search.\"\n )\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_spaces)} spaces in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(\n f\"[SEARCH] Embedding spaces detected in index: \"\n f\"{[space.space_id for space in available_spaces]}\"\n )\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for space in available_spaces:\n model_name = space.space_id\n try:\n emb_obj = None\n\n # The OpenRAG proxy adapter can create an immutable child for\n # any provider-qualified or legacy route. Resolve through it\n # first so a legacy OpenAI vector is never queried through the\n # currently selected Azure/Watsonx provider by accident.\n for candidate in embeddings_list:\n resolver = getattr(candidate, \"for_model\", None)\n if callable(resolver):\n emb_obj = resolver(space.route_model)\n break\n\n # Backward compatibility for custom flows that still wire one\n # concrete embedding adapter per model.\n if emb_obj is None:\n for identifier in (space.space_id, space.field_identity):\n if identifier in embedding_by_model:\n emb_obj = embedding_by_model[identifier]\n break\n\n if emb_obj is not None:\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[space.field_identity] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except Exception as e: # provider clients expose different exception families\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n self.log(\n \"[WARN] No indexed embedding space could be resolved; \"\n \"continuing with keyword-only search\"\n )\n\n index_properties = self._get_index_properties(client)\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Continuing with keyword-only search.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index; using keyword search.\"\n )\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n keyword_query = {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3 if knn_queries_with_candidates else 1.0,\n }\n }\n should_queries: list[dict[str, Any]] = [keyword_query]\n if knn_queries_with_candidates:\n should_queries.insert(\n 0,\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0,\n \"boost\": 0.7,\n \"queries\": knn_queries_with_candidates,\n }\n },\n )\n\n # Keyword search intentionally has no embedding-field existence filter:\n # documents in an unresolved legacy space must remain discoverable.\n body = {\n \"query\": {\n \"bool\": {\n \"should\": should_queries,\n \"minimum_should_match\": 1,\n \"filter\": filter_clauses,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"embedding_provider\",\n \"embedding_space_id\",\n \"parser\",\n \"chunk_size\",\n \"chunk_overlap\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing hybrid search with {len(knn_queries_with_candidates)} resolved embedding spaces: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if knn_queries_with_candidates and use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif knn_queries_with_candidates and (\n \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered)\n ):\n logger.warning(\n \"KNN search failed for an indexed embedding field; retrying keyword-only\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"should\"] = [keyword_query]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"spaces_in_index={[space.space_id for space in available_spaces]}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": fence_untrusted_text(hit[\"_source\"].get(\"text\", \"\")),\n \"metadata\": {\n **{k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"chunk_id\": hit.get(\"_id\"),\n \"id\": hit.get(\"_id\"),\n \"score\": hit.get(\"_score\"),\n },\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as a list of Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in a format that downstream Parser\n components and tool wrappers can consume directly without DataFrame string truncation.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n List containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return raw_list\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"+ "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport unicodedata\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom dataclasses import dataclass\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\nEMBEDDING_SPACE_PAGE_SIZE = 100\nSELECTED_EMBEDDING_PROVIDER_VAR = \"SELECTED_EMBEDDING_PROVIDER\"\nLEGACY_EMBEDDING_ROUTE_PREFIX = \"legacy:\"\nINDEXED_EMBEDDING_ROUTE_PREFIX = \"space:\"\n\n# Some PDF fonts map common kanji (日, 月, 木, 金, 土, ...) to the look-alike\n# Kangxi Radical code points in their /ToUnicode CMap, and Docling copies them\n# through unchanged. Map just those blocks back to CJK Unified Ideographs;\n# full NFKC would also rewrite full-width digits, ㈱, ①, half-width katakana.\n# Mirrors normalize_cjk_radicals in src/utils/document_processing.py.\nCJK_RADICAL_TABLE = {\n cp: unicodedata.normalize(\"NFKC\", chr(cp))\n for cp in range(0x2E80, 0x2FE0) # CJK Radicals Supplement + Kangxi Radicals\n if unicodedata.normalize(\"NFKC\", chr(cp)) != chr(cp)\n}\n\n\n@dataclass(frozen=True)\nclass EmbeddingSpace:\n \"\"\"One indexed vector space and the model route needed to query it.\"\"\"\n\n space_id: str\n route_model: str\n field_identity: str\n legacy: bool = False\n\n\ndef build_embedding_space_aggregation(\n *,\n size: int,\n qualified_after: dict[str, Any] | None = None,\n legacy_after: dict[str, Any] | None = None,\n include_qualified: bool = True,\n include_legacy: bool = True,\n) -> dict[str, Any]:\n \"\"\"Build pageable discovery aggregations for exact and legacy vector spaces.\"\"\"\n aggregations: dict[str, Any] = {}\n if include_qualified:\n composite: dict[str, Any] = {\n \"size\": size,\n \"sources\": [{\"space_id\": {\"terms\": {\"field\": \"embedding_space_id\"}}}],\n }\n if qualified_after:\n composite[\"after\"] = qualified_after\n aggregations[\"embedding_spaces\"] = {\"composite\": composite}\n\n if include_legacy:\n composite = {\n \"size\": size,\n \"sources\": [{\"model\": {\"terms\": {\"field\": \"embedding_model\"}}}],\n }\n if legacy_after:\n composite[\"after\"] = legacy_after\n aggregations[\"legacy_embedding_models\"] = {\"composite\": composite}\n return aggregations\n\n\ndef embedding_space_after_keys(\n result: dict[str, Any],\n) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:\n \"\"\"Return composite pagination cursors for exact and legacy aggregations.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_after = aggregations.get(\"embedding_spaces\", {}).get(\"after_key\")\n legacy_after = aggregations.get(\"legacy_embedding_models\", {}).get(\"after_key\")\n return qualified_after, legacy_after\n\n\ndef embedding_spaces_from_aggregation(result: dict[str, Any]) -> list[EmbeddingSpace]:\n \"\"\"Convert an OpenSearch aggregation response into stable retrieval identities.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_buckets = aggregations.get(\"embedding_spaces\", {}).get(\"buckets\", [])\n legacy_buckets = aggregations.get(\"legacy_embedding_models\", {}).get(\"buckets\", [])\n\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n for bucket in qualified_buckets:\n key = bucket.get(\"key\")\n raw_space_id = key.get(\"space_id\") if isinstance(key, dict) else key\n space_id = str(raw_space_id or \"\").strip()\n if not space_id or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=f\"{INDEXED_EMBEDDING_ROUTE_PREFIX}{space_id}\",\n field_identity=space_id,\n )\n )\n\n for bucket in legacy_buckets:\n key = bucket.get(\"key\")\n raw_model = key.get(\"model\") if isinstance(key, dict) else key\n model = str(raw_model or \"\").strip()\n space_id = f\"{LEGACY_EMBEDDING_ROUTE_PREFIX}{model}\"\n if not model or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=space_id,\n field_identity=model,\n legacy=True,\n )\n )\n return spaces\n\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {\n h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None\n }\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\nUNTRUSTED_CHUNK_FENCE_START = \"<<<UNTRUSTED_DOC_CHUNK>>>\"\nUNTRUSTED_CHUNK_FENCE_END = \"<<<END_UNTRUSTED_DOC_CHUNK>>>\"\n\n\ndef fence_untrusted_text(text: str) -> str:\n \"\"\"Wrap retrieved document text in delimiters marking it as untrusted data.\n\n Retrieved chunk text can contain attacker-controlled natural-language\n instructions (indirect prompt injection, VULN-13906). Fencing it lets the\n system prompt tell the LLM to treat anything between these markers as\n data only, never as instructions or tool directives.\n\n Any literal fence markers already present in `text` are escaped first, so\n a poisoned chunk can't embed a fake end-of-fence marker to break out of\n the untrusted section and have its continuation misread as trusted.\n \"\"\"\n if not text:\n return text\n escaped = text.replace(UNTRUSTED_CHUNK_FENCE_START, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_START).replace(\n UNTRUSTED_CHUNK_FENCE_END, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_END\n )\n return f\"{UNTRUSTED_CHUNK_FENCE_START}\\n{escaped}\\n{UNTRUSTED_CHUNK_FENCE_END}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"embedding_provider_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"embedding_provider_name\",\n display_name=\"Embedding Provider\",\n value=SELECTED_EMBEDDING_PROVIDER_VAR,\n info=(\n \"Exact OpenRAG provider key used for new ingestion metadata and vector fields. \"\n \"Bound to SELECTED_EMBEDDING_PROVIDER in managed OpenRAG flows.\"\n ),\n load_from_db=True,\n advanced=True,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"OPENSEARCH_PASSWORD\",\n load_from_db=True,\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False,\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n SecretStrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n if \"text\" in source:\n source[\"text\"] = fence_untrusted_text(source[\"text\"])\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n\n logger.debug(\n \"[OpenRAG callback config] configured for ingest_run_id=%s (url_len=%d, token_len=%d)\",\n ingest_run_id,\n len(url),\n len(token),\n )\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n logger.debug(\n f\"[OpenRAG ingest POST] total_chunks={len(requests)} batch_size={batch_size} timeout_s={timeout}\"\n )\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.debug(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n logger.debug(\n f\"[OpenRAG ingest POST resp] batch={batch_number} status={response.status_code}\"\n )\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code})\"\n )\n logger.warning(msg)\n raise RuntimeError(msg)\n\n logger.debug(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n embedding_provider: str | None = None,\n embedding_space_id: str | None = None,\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n embedding_provider: Exact provider key, when known\n embedding_space_id: Provider-qualified vector-space identity\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if embedding_provider:\n request[\"embedding_provider\"] = embedding_provider\n if embedding_space_id:\n request[\"embedding_space_id\"] = embedding_space_id\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", url),\n (\"openrag_ingest_token\", token),\n (\"openrag_ingest_run_id\", ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self._openrag_input_to_str(self.password) or \"\").strip()\n if pwd == \"OPENSEARCH_PASSWORD\" or not pwd:\n from config.settings import get_opensearch_password\n\n pwd = get_opensearch_password() or \"\"\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n embedding_provider = self._openrag_input_to_str(\n getattr(self, \"embedding_provider_name\", \"\")\n ).lower()\n if embedding_provider == SELECTED_EMBEDDING_PROVIDER_VAR.lower():\n embedding_provider = \"\"\n embedding_space_id = (\n f\"{embedding_provider}:{embedding_model}\" if embedding_provider else None\n )\n dynamic_field_name = get_embedding_field_name(embedding_space_id or embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n if isinstance(text, str):\n text = text.translate(CJK_RADICAL_TABLE)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n embedding_provider=embedding_provider or None,\n embedding_space_id=embedding_space_id,\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(\n self, client: OpenSearch, filter_clauses: list[dict] | None = None\n ) -> list[EmbeddingSpace]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n result: dict[str, Any] = {}\n\n # Composite aggregations must be top-level. Exact and legacy\n # spaces use different document filters, so paginate them in two\n # independent passes rather than nesting composite under filter.\n for legacy in (False, True):\n after = None\n while True:\n agg_query = {\n \"size\": 0,\n \"aggs\": build_embedding_space_aggregation(\n size=EMBEDDING_SPACE_PAGE_SIZE,\n qualified_after=None if legacy else after,\n legacy_after=after if legacy else None,\n include_qualified=not legacy,\n include_legacy=legacy,\n ),\n }\n bool_query: dict[str, Any] = {}\n if filter_clauses:\n bool_query[\"filter\"] = filter_clauses\n if legacy:\n bool_query[\"must_not\"] = [{\"exists\": {\"field\": \"embedding_space_id\"}}]\n if bool_query:\n agg_query[\"query\"] = {\"bool\": bool_query}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n for space in embedding_spaces_from_aggregation(result):\n if space.space_id not in seen:\n seen.add(space.space_id)\n spaces.append(space)\n\n qualified_after, legacy_after = embedding_space_after_keys(result)\n next_after = legacy_after if legacy else qualified_after\n if not next_after or next_after == after:\n break\n after = next_after\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding spaces in corpus: {[space.space_id for space in spaces]}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not spaces:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Provider provenance cannot be inferred safely after discovery\n # fails. Keyword search remains available without guessing.\n return []\n else:\n return spaces\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_spaces = self._detect_available_models(client, filter_clauses)\n\n if not available_spaces:\n logger.warning(\"No embedding spaces found in index; using keyword-only search\")\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.warning(\n \"No valid embedding adapter is available; continuing with keyword-only search.\"\n )\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_spaces)} spaces in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(\n f\"[SEARCH] Embedding spaces detected in index: \"\n f\"{[space.space_id for space in available_spaces]}\"\n )\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for space in available_spaces:\n model_name = space.space_id\n try:\n emb_obj = None\n\n # The OpenRAG proxy adapter can create an immutable child for\n # any provider-qualified or legacy route. Resolve through it\n # first so a legacy OpenAI vector is never queried through the\n # currently selected Azure/Watsonx provider by accident.\n for candidate in embeddings_list:\n resolver = getattr(candidate, \"for_model\", None)\n if callable(resolver):\n emb_obj = resolver(space.route_model)\n break\n\n # Backward compatibility for custom flows that still wire one\n # concrete embedding adapter per model.\n if emb_obj is None:\n for identifier in (space.space_id, space.field_identity):\n if identifier in embedding_by_model:\n emb_obj = embedding_by_model[identifier]\n break\n\n if emb_obj is not None:\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[space.field_identity] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except Exception as e: # provider clients expose different exception families\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n self.log(\n \"[WARN] No indexed embedding space could be resolved; \"\n \"continuing with keyword-only search\"\n )\n\n index_properties = self._get_index_properties(client)\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Continuing with keyword-only search.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index; using keyword search.\"\n )\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n keyword_query = {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3 if knn_queries_with_candidates else 1.0,\n }\n }\n should_queries: list[dict[str, Any]] = [keyword_query]\n if knn_queries_with_candidates:\n should_queries.insert(\n 0,\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0,\n \"boost\": 0.7,\n \"queries\": knn_queries_with_candidates,\n }\n },\n )\n\n # Keyword search intentionally has no embedding-field existence filter:\n # documents in an unresolved legacy space must remain discoverable.\n body = {\n \"query\": {\n \"bool\": {\n \"should\": should_queries,\n \"minimum_should_match\": 1,\n \"filter\": filter_clauses,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"embedding_provider\",\n \"embedding_space_id\",\n \"parser\",\n \"chunk_size\",\n \"chunk_overlap\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing hybrid search with {len(knn_queries_with_candidates)} resolved embedding spaces: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if knn_queries_with_candidates and use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif knn_queries_with_candidates and (\n \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered)\n ):\n logger.warning(\n \"KNN search failed for an indexed embedding field; retrying keyword-only\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"should\"] = [keyword_query]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"spaces_in_index={[space.space_id for space in available_spaces]}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": fence_untrusted_text(hit[\"_source\"].get(\"text\", \"\")),\n \"metadata\": {\n **{k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"chunk_id\": hit.get(\"_id\"),\n \"id\": hit.get(\"_id\"),\n \"score\": hit.get(\"_score\"),\n },\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as a list of Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in a format that downstream Parser\n components and tool wrappers can consume directly without DataFrame string truncation.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n List containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return raw_list\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"},"docs_metadata": {"_input_type": "TableInput",diff --git a/flows/openrag_agent.json b/flows/openrag_agent.jsonindex 838abaccc..a269c4e62 100644--- a/flows/openrag_agent.json+++ b/flows/openrag_agent.json@@ -2738,7 +2738,7 @@"last_updated": "2026-09-16T13:50:51.381Z","legacy": false,"metadata": {- "code_hash": "a3727d7c66eb",+ "code_hash": "1a2a5b8d0ca7","dependencies": {"dependencies": [{@@ -2865,7 +2865,7 @@"show": true,"title_case": false,"type": "code",- "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom dataclasses import dataclass\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\nEMBEDDING_SPACE_PAGE_SIZE = 100\nSELECTED_EMBEDDING_PROVIDER_VAR = \"SELECTED_EMBEDDING_PROVIDER\"\nLEGACY_EMBEDDING_ROUTE_PREFIX = \"legacy:\"\nINDEXED_EMBEDDING_ROUTE_PREFIX = \"space:\"\n\n\n@dataclass(frozen=True)\nclass EmbeddingSpace:\n \"\"\"One indexed vector space and the model route needed to query it.\"\"\"\n\n space_id: str\n route_model: str\n field_identity: str\n legacy: bool = False\n\n\ndef build_embedding_space_aggregation(\n *,\n size: int,\n qualified_after: dict[str, Any] | None = None,\n legacy_after: dict[str, Any] | None = None,\n include_qualified: bool = True,\n include_legacy: bool = True,\n) -> dict[str, Any]:\n \"\"\"Build pageable discovery aggregations for exact and legacy vector spaces.\"\"\"\n aggregations: dict[str, Any] = {}\n if include_qualified:\n composite: dict[str, Any] = {\n \"size\": size,\n \"sources\": [{\"space_id\": {\"terms\": {\"field\": \"embedding_space_id\"}}}],\n }\n if qualified_after:\n composite[\"after\"] = qualified_after\n aggregations[\"embedding_spaces\"] = {\"composite\": composite}\n\n if include_legacy:\n composite = {\n \"size\": size,\n \"sources\": [{\"model\": {\"terms\": {\"field\": \"embedding_model\"}}}],\n }\n if legacy_after:\n composite[\"after\"] = legacy_after\n aggregations[\"legacy_embedding_models\"] = {\"composite\": composite}\n return aggregations\n\n\ndef embedding_space_after_keys(\n result: dict[str, Any],\n) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:\n \"\"\"Return composite pagination cursors for exact and legacy aggregations.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_after = aggregations.get(\"embedding_spaces\", {}).get(\"after_key\")\n legacy_after = aggregations.get(\"legacy_embedding_models\", {}).get(\"after_key\")\n return qualified_after, legacy_after\n\n\ndef embedding_spaces_from_aggregation(result: dict[str, Any]) -> list[EmbeddingSpace]:\n \"\"\"Convert an OpenSearch aggregation response into stable retrieval identities.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_buckets = aggregations.get(\"embedding_spaces\", {}).get(\"buckets\", [])\n legacy_buckets = aggregations.get(\"legacy_embedding_models\", {}).get(\"buckets\", [])\n\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n for bucket in qualified_buckets:\n key = bucket.get(\"key\")\n raw_space_id = key.get(\"space_id\") if isinstance(key, dict) else key\n space_id = str(raw_space_id or \"\").strip()\n if not space_id or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=f\"{INDEXED_EMBEDDING_ROUTE_PREFIX}{space_id}\",\n field_identity=space_id,\n )\n )\n\n for bucket in legacy_buckets:\n key = bucket.get(\"key\")\n raw_model = key.get(\"model\") if isinstance(key, dict) else key\n model = str(raw_model or \"\").strip()\n space_id = f\"{LEGACY_EMBEDDING_ROUTE_PREFIX}{model}\"\n if not model or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=space_id,\n field_identity=model,\n legacy=True,\n )\n )\n return spaces\n\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {\n h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None\n }\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\nUNTRUSTED_CHUNK_FENCE_START = \"<<<UNTRUSTED_DOC_CHUNK>>>\"\nUNTRUSTED_CHUNK_FENCE_END = \"<<<END_UNTRUSTED_DOC_CHUNK>>>\"\n\n\ndef fence_untrusted_text(text: str) -> str:\n \"\"\"Wrap retrieved document text in delimiters marking it as untrusted data.\n\n Retrieved chunk text can contain attacker-controlled natural-language\n instructions (indirect prompt injection, VULN-13906). Fencing it lets the\n system prompt tell the LLM to treat anything between these markers as\n data only, never as instructions or tool directives.\n\n Any literal fence markers already present in `text` are escaped first, so\n a poisoned chunk can't embed a fake end-of-fence marker to break out of\n the untrusted section and have its continuation misread as trusted.\n \"\"\"\n if not text:\n return text\n escaped = text.replace(UNTRUSTED_CHUNK_FENCE_START, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_START).replace(\n UNTRUSTED_CHUNK_FENCE_END, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_END\n )\n return f\"{UNTRUSTED_CHUNK_FENCE_START}\\n{escaped}\\n{UNTRUSTED_CHUNK_FENCE_END}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"embedding_provider_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"embedding_provider_name\",\n display_name=\"Embedding Provider\",\n value=SELECTED_EMBEDDING_PROVIDER_VAR,\n info=(\n \"Exact OpenRAG provider key used for new ingestion metadata and vector fields. \"\n \"Bound to SELECTED_EMBEDDING_PROVIDER in managed OpenRAG flows.\"\n ),\n load_from_db=True,\n advanced=True,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"OPENSEARCH_PASSWORD\",\n load_from_db=True,\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False,\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n SecretStrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n if \"text\" in source:\n source[\"text\"] = fence_untrusted_text(source[\"text\"])\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n\n logger.debug(\n \"[OpenRAG callback config] configured for ingest_run_id=%s (url_len=%d, token_len=%d)\",\n ingest_run_id,\n len(url),\n len(token),\n )\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n logger.debug(\n f\"[OpenRAG ingest POST] total_chunks={len(requests)} batch_size={batch_size} timeout_s={timeout}\"\n )\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.debug(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n logger.debug(\n f\"[OpenRAG ingest POST resp] batch={batch_number} status={response.status_code}\"\n )\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code})\"\n )\n logger.warning(msg)\n raise RuntimeError(msg)\n\n logger.debug(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n embedding_provider: str | None = None,\n embedding_space_id: str | None = None,\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n embedding_provider: Exact provider key, when known\n embedding_space_id: Provider-qualified vector-space identity\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if embedding_provider:\n request[\"embedding_provider\"] = embedding_provider\n if embedding_space_id:\n request[\"embedding_space_id\"] = embedding_space_id\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", url),\n (\"openrag_ingest_token\", token),\n (\"openrag_ingest_run_id\", ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self._openrag_input_to_str(self.password) or \"\").strip()\n if pwd == \"OPENSEARCH_PASSWORD\" or not pwd:\n from config.settings import get_opensearch_password\n\n pwd = get_opensearch_password() or \"\"\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n embedding_provider = self._openrag_input_to_str(\n getattr(self, \"embedding_provider_name\", \"\")\n ).lower()\n if embedding_provider == SELECTED_EMBEDDING_PROVIDER_VAR.lower():\n embedding_provider = \"\"\n embedding_space_id = (\n f\"{embedding_provider}:{embedding_model}\" if embedding_provider else None\n )\n dynamic_field_name = get_embedding_field_name(embedding_space_id or embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n embedding_provider=embedding_provider or None,\n embedding_space_id=embedding_space_id,\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(\n self, client: OpenSearch, filter_clauses: list[dict] | None = None\n ) -> list[EmbeddingSpace]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n result: dict[str, Any] = {}\n\n # Composite aggregations must be top-level. Exact and legacy\n # spaces use different document filters, so paginate them in two\n # independent passes rather than nesting composite under filter.\n for legacy in (False, True):\n after = None\n while True:\n agg_query = {\n \"size\": 0,\n \"aggs\": build_embedding_space_aggregation(\n size=EMBEDDING_SPACE_PAGE_SIZE,\n qualified_after=None if legacy else after,\n legacy_after=after if legacy else None,\n include_qualified=not legacy,\n include_legacy=legacy,\n ),\n }\n bool_query: dict[str, Any] = {}\n if filter_clauses:\n bool_query[\"filter\"] = filter_clauses\n if legacy:\n bool_query[\"must_not\"] = [{\"exists\": {\"field\": \"embedding_space_id\"}}]\n if bool_query:\n agg_query[\"query\"] = {\"bool\": bool_query}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n for space in embedding_spaces_from_aggregation(result):\n if space.space_id not in seen:\n seen.add(space.space_id)\n spaces.append(space)\n\n qualified_after, legacy_after = embedding_space_after_keys(result)\n next_after = legacy_after if legacy else qualified_after\n if not next_after or next_after == after:\n break\n after = next_after\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding spaces in corpus: {[space.space_id for space in spaces]}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not spaces:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Provider provenance cannot be inferred safely after discovery\n # fails. Keyword search remains available without guessing.\n return []\n else:\n return spaces\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_spaces = self._detect_available_models(client, filter_clauses)\n\n if not available_spaces:\n logger.warning(\"No embedding spaces found in index; using keyword-only search\")\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.warning(\n \"No valid embedding adapter is available; continuing with keyword-only search.\"\n )\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_spaces)} spaces in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(\n f\"[SEARCH] Embedding spaces detected in index: \"\n f\"{[space.space_id for space in available_spaces]}\"\n )\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for space in available_spaces:\n model_name = space.space_id\n try:\n emb_obj = None\n\n # The OpenRAG proxy adapter can create an immutable child for\n # any provider-qualified or legacy route. Resolve through it\n # first so a legacy OpenAI vector is never queried through the\n # currently selected Azure/Watsonx provider by accident.\n for candidate in embeddings_list:\n resolver = getattr(candidate, \"for_model\", None)\n if callable(resolver):\n emb_obj = resolver(space.route_model)\n break\n\n # Backward compatibility for custom flows that still wire one\n # concrete embedding adapter per model.\n if emb_obj is None:\n for identifier in (space.space_id, space.field_identity):\n if identifier in embedding_by_model:\n emb_obj = embedding_by_model[identifier]\n break\n\n if emb_obj is not None:\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[space.field_identity] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except Exception as e: # provider clients expose different exception families\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n self.log(\n \"[WARN] No indexed embedding space could be resolved; \"\n \"continuing with keyword-only search\"\n )\n\n index_properties = self._get_index_properties(client)\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Continuing with keyword-only search.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index; using keyword search.\"\n )\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n keyword_query = {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3 if knn_queries_with_candidates else 1.0,\n }\n }\n should_queries: list[dict[str, Any]] = [keyword_query]\n if knn_queries_with_candidates:\n should_queries.insert(\n 0,\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0,\n \"boost\": 0.7,\n \"queries\": knn_queries_with_candidates,\n }\n },\n )\n\n # Keyword search intentionally has no embedding-field existence filter:\n # documents in an unresolved legacy space must remain discoverable.\n body = {\n \"query\": {\n \"bool\": {\n \"should\": should_queries,\n \"minimum_should_match\": 1,\n \"filter\": filter_clauses,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"embedding_provider\",\n \"embedding_space_id\",\n \"parser\",\n \"chunk_size\",\n \"chunk_overlap\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing hybrid search with {len(knn_queries_with_candidates)} resolved embedding spaces: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if knn_queries_with_candidates and use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif knn_queries_with_candidates and (\n \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered)\n ):\n logger.warning(\n \"KNN search failed for an indexed embedding field; retrying keyword-only\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"should\"] = [keyword_query]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"spaces_in_index={[space.space_id for space in available_spaces]}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": fence_untrusted_text(hit[\"_source\"].get(\"text\", \"\")),\n \"metadata\": {\n **{k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"chunk_id\": hit.get(\"_id\"),\n \"id\": hit.get(\"_id\"),\n \"score\": hit.get(\"_score\"),\n },\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as a list of Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in a format that downstream Parser\n components and tool wrappers can consume directly without DataFrame string truncation.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n List containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return raw_list\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"+ "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport unicodedata\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom dataclasses import dataclass\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\nEMBEDDING_SPACE_PAGE_SIZE = 100\nSELECTED_EMBEDDING_PROVIDER_VAR = \"SELECTED_EMBEDDING_PROVIDER\"\nLEGACY_EMBEDDING_ROUTE_PREFIX = \"legacy:\"\nINDEXED_EMBEDDING_ROUTE_PREFIX = \"space:\"\n\n# Some PDF fonts map common kanji (日, 月, 木, 金, 土, ...) to the look-alike\n# Kangxi Radical code points in their /ToUnicode CMap, and Docling copies them\n# through unchanged. Map just those blocks back to CJK Unified Ideographs;\n# full NFKC would also rewrite full-width digits, ㈱, ①, half-width katakana.\n# Mirrors normalize_cjk_radicals in src/utils/document_processing.py.\nCJK_RADICAL_TABLE = {\n cp: unicodedata.normalize(\"NFKC\", chr(cp))\n for cp in range(0x2E80, 0x2FE0) # CJK Radicals Supplement + Kangxi Radicals\n if unicodedata.normalize(\"NFKC\", chr(cp)) != chr(cp)\n}\n\n\n@dataclass(frozen=True)\nclass EmbeddingSpace:\n \"\"\"One indexed vector space and the model route needed to query it.\"\"\"\n\n space_id: str\n route_model: str\n field_identity: str\n legacy: bool = False\n\n\ndef build_embedding_space_aggregation(\n *,\n size: int,\n qualified_after: dict[str, Any] | None = None,\n legacy_after: dict[str, Any] | None = None,\n include_qualified: bool = True,\n include_legacy: bool = True,\n) -> dict[str, Any]:\n \"\"\"Build pageable discovery aggregations for exact and legacy vector spaces.\"\"\"\n aggregations: dict[str, Any] = {}\n if include_qualified:\n composite: dict[str, Any] = {\n \"size\": size,\n \"sources\": [{\"space_id\": {\"terms\": {\"field\": \"embedding_space_id\"}}}],\n }\n if qualified_after:\n composite[\"after\"] = qualified_after\n aggregations[\"embedding_spaces\"] = {\"composite\": composite}\n\n if include_legacy:\n composite = {\n \"size\": size,\n \"sources\": [{\"model\": {\"terms\": {\"field\": \"embedding_model\"}}}],\n }\n if legacy_after:\n composite[\"after\"] = legacy_after\n aggregations[\"legacy_embedding_models\"] = {\"composite\": composite}\n return aggregations\n\n\ndef embedding_space_after_keys(\n result: dict[str, Any],\n) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:\n \"\"\"Return composite pagination cursors for exact and legacy aggregations.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_after = aggregations.get(\"embedding_spaces\", {}).get(\"after_key\")\n legacy_after = aggregations.get(\"legacy_embedding_models\", {}).get(\"after_key\")\n return qualified_after, legacy_after\n\n\ndef embedding_spaces_from_aggregation(result: dict[str, Any]) -> list[EmbeddingSpace]:\n \"\"\"Convert an OpenSearch aggregation response into stable retrieval identities.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_buckets = aggregations.get(\"embedding_spaces\", {}).get(\"buckets\", [])\n legacy_buckets = aggregations.get(\"legacy_embedding_models\", {}).get(\"buckets\", [])\n\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n for bucket in qualified_buckets:\n key = bucket.get(\"key\")\n raw_space_id = key.get(\"space_id\") if isinstance(key, dict) else key\n space_id = str(raw_space_id or \"\").strip()\n if not space_id or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=f\"{INDEXED_EMBEDDING_ROUTE_PREFIX}{space_id}\",\n field_identity=space_id,\n )\n )\n\n for bucket in legacy_buckets:\n key = bucket.get(\"key\")\n raw_model = key.get(\"model\") if isinstance(key, dict) else key\n model = str(raw_model or \"\").strip()\n space_id = f\"{LEGACY_EMBEDDING_ROUTE_PREFIX}{model}\"\n if not model or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=space_id,\n field_identity=model,\n legacy=True,\n )\n )\n return spaces\n\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {\n h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None\n }\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\nUNTRUSTED_CHUNK_FENCE_START = \"<<<UNTRUSTED_DOC_CHUNK>>>\"\nUNTRUSTED_CHUNK_FENCE_END = \"<<<END_UNTRUSTED_DOC_CHUNK>>>\"\n\n\ndef fence_untrusted_text(text: str) -> str:\n \"\"\"Wrap retrieved document text in delimiters marking it as untrusted data.\n\n Retrieved chunk text can contain attacker-controlled natural-language\n instructions (indirect prompt injection, VULN-13906). Fencing it lets the\n system prompt tell the LLM to treat anything between these markers as\n data only, never as instructions or tool directives.\n\n Any literal fence markers already present in `text` are escaped first, so\n a poisoned chunk can't embed a fake end-of-fence marker to break out of\n the untrusted section and have its continuation misread as trusted.\n \"\"\"\n if not text:\n return text\n escaped = text.replace(UNTRUSTED_CHUNK_FENCE_START, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_START).replace(\n UNTRUSTED_CHUNK_FENCE_END, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_END\n )\n return f\"{UNTRUSTED_CHUNK_FENCE_START}\\n{escaped}\\n{UNTRUSTED_CHUNK_FENCE_END}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"embedding_provider_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"embedding_provider_name\",\n display_name=\"Embedding Provider\",\n value=SELECTED_EMBEDDING_PROVIDER_VAR,\n info=(\n \"Exact OpenRAG provider key used for new ingestion metadata and vector fields. \"\n \"Bound to SELECTED_EMBEDDING_PROVIDER in managed OpenRAG flows.\"\n ),\n load_from_db=True,\n advanced=True,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"OPENSEARCH_PASSWORD\",\n load_from_db=True,\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False,\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n SecretStrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n if \"text\" in source:\n source[\"text\"] = fence_untrusted_text(source[\"text\"])\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n\n logger.debug(\n \"[OpenRAG callback config] configured for ingest_run_id=%s (url_len=%d, token_len=%d)\",\n ingest_run_id,\n len(url),\n len(token),\n )\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n logger.debug(\n f\"[OpenRAG ingest POST] total_chunks={len(requests)} batch_size={batch_size} timeout_s={timeout}\"\n )\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.debug(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n logger.debug(\n f\"[OpenRAG ingest POST resp] batch={batch_number} status={response.status_code}\"\n )\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code})\"\n )\n logger.warning(msg)\n raise RuntimeError(msg)\n\n logger.debug(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n embedding_provider: str | None = None,\n embedding_space_id: str | None = None,\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n embedding_provider: Exact provider key, when known\n embedding_space_id: Provider-qualified vector-space identity\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if embedding_provider:\n request[\"embedding_provider\"] = embedding_provider\n if embedding_space_id:\n request[\"embedding_space_id\"] = embedding_space_id\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", url),\n (\"openrag_ingest_token\", token),\n (\"openrag_ingest_run_id\", ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self._openrag_input_to_str(self.password) or \"\").strip()\n if pwd == \"OPENSEARCH_PASSWORD\" or not pwd:\n from config.settings import get_opensearch_password\n\n pwd = get_opensearch_password() or \"\"\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n embedding_provider = self._openrag_input_to_str(\n getattr(self, \"embedding_provider_name\", \"\")\n ).lower()\n if embedding_provider == SELECTED_EMBEDDING_PROVIDER_VAR.lower():\n embedding_provider = \"\"\n embedding_space_id = (\n f\"{embedding_provider}:{embedding_model}\" if embedding_provider else None\n )\n dynamic_field_name = get_embedding_field_name(embedding_space_id or embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n if isinstance(text, str):\n text = text.translate(CJK_RADICAL_TABLE)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n embedding_provider=embedding_provider or None,\n embedding_space_id=embedding_space_id,\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(\n self, client: OpenSearch, filter_clauses: list[dict] | None = None\n ) -> list[EmbeddingSpace]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n result: dict[str, Any] = {}\n\n # Composite aggregations must be top-level. Exact and legacy\n # spaces use different document filters, so paginate them in two\n # independent passes rather than nesting composite under filter.\n for legacy in (False, True):\n after = None\n while True:\n agg_query = {\n \"size\": 0,\n \"aggs\": build_embedding_space_aggregation(\n size=EMBEDDING_SPACE_PAGE_SIZE,\n qualified_after=None if legacy else after,\n legacy_after=after if legacy else None,\n include_qualified=not legacy,\n include_legacy=legacy,\n ),\n }\n bool_query: dict[str, Any] = {}\n if filter_clauses:\n bool_query[\"filter\"] = filter_clauses\n if legacy:\n bool_query[\"must_not\"] = [{\"exists\": {\"field\": \"embedding_space_id\"}}]\n if bool_query:\n agg_query[\"query\"] = {\"bool\": bool_query}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n for space in embedding_spaces_from_aggregation(result):\n if space.space_id not in seen:\n seen.add(space.space_id)\n spaces.append(space)\n\n qualified_after, legacy_after = embedding_space_after_keys(result)\n next_after = legacy_after if legacy else qualified_after\n if not next_after or next_after == after:\n break\n after = next_after\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding spaces in corpus: {[space.space_id for space in spaces]}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not spaces:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Provider provenance cannot be inferred safely after discovery\n # fails. Keyword search remains available without guessing.\n return []\n else:\n return spaces\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_spaces = self._detect_available_models(client, filter_clauses)\n\n if not available_spaces:\n logger.warning(\"No embedding spaces found in index; using keyword-only search\")\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.warning(\n \"No valid embedding adapter is available; continuing with keyword-only search.\"\n )\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_spaces)} spaces in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(\n f\"[SEARCH] Embedding spaces detected in index: \"\n f\"{[space.space_id for space in available_spaces]}\"\n )\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for space in available_spaces:\n model_name = space.space_id\n try:\n emb_obj = None\n\n # The OpenRAG proxy adapter can create an immutable child for\n # any provider-qualified or legacy route. Resolve through it\n # first so a legacy OpenAI vector is never queried through the\n # currently selected Azure/Watsonx provider by accident.\n for candidate in embeddings_list:\n resolver = getattr(candidate, \"for_model\", None)\n if callable(resolver):\n emb_obj = resolver(space.route_model)\n break\n\n # Backward compatibility for custom flows that still wire one\n # concrete embedding adapter per model.\n if emb_obj is None:\n for identifier in (space.space_id, space.field_identity):\n if identifier in embedding_by_model:\n emb_obj = embedding_by_model[identifier]\n break\n\n if emb_obj is not None:\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[space.field_identity] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except Exception as e: # provider clients expose different exception families\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n self.log(\n \"[WARN] No indexed embedding space could be resolved; \"\n \"continuing with keyword-only search\"\n )\n\n index_properties = self._get_index_properties(client)\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Continuing with keyword-only search.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index; using keyword search.\"\n )\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n keyword_query = {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3 if knn_queries_with_candidates else 1.0,\n }\n }\n should_queries: list[dict[str, Any]] = [keyword_query]\n if knn_queries_with_candidates:\n should_queries.insert(\n 0,\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0,\n \"boost\": 0.7,\n \"queries\": knn_queries_with_candidates,\n }\n },\n )\n\n # Keyword search intentionally has no embedding-field existence filter:\n # documents in an unresolved legacy space must remain discoverable.\n body = {\n \"query\": {\n \"bool\": {\n \"should\": should_queries,\n \"minimum_should_match\": 1,\n \"filter\": filter_clauses,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"embedding_provider\",\n \"embedding_space_id\",\n \"parser\",\n \"chunk_size\",\n \"chunk_overlap\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing hybrid search with {len(knn_queries_with_candidates)} resolved embedding spaces: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if knn_queries_with_candidates and use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif knn_queries_with_candidates and (\n \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered)\n ):\n logger.warning(\n \"KNN search failed for an indexed embedding field; retrying keyword-only\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"should\"] = [keyword_query]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"spaces_in_index={[space.space_id for space in available_spaces]}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": fence_untrusted_text(hit[\"_source\"].get(\"text\", \"\")),\n \"metadata\": {\n **{k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"chunk_id\": hit.get(\"_id\"),\n \"id\": hit.get(\"_id\"),\n \"score\": hit.get(\"_score\"),\n },\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as a list of Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in a format that downstream Parser\n components and tool wrappers can consume directly without DataFrame string truncation.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n List containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return raw_list\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"},"docs_metadata": {"_input_type": "TableInput",diff --git a/flows/openrag_nudges.json b/flows/openrag_nudges.jsonindex 9219cafb8..111086f0d 100644--- a/flows/openrag_nudges.json+++ b/flows/openrag_nudges.json@@ -1460,7 +1460,7 @@"last_updated": "2026-08-05T16:19:12.028Z","legacy": false,"metadata": {- "code_hash": "a3727d7c66eb",+ "code_hash": "1a2a5b8d0ca7","dependencies": {"dependencies": [{@@ -1623,7 +1623,7 @@"show": true,"title_case": false,"type": "code",- "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom dataclasses import dataclass\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\nEMBEDDING_SPACE_PAGE_SIZE = 100\nSELECTED_EMBEDDING_PROVIDER_VAR = \"SELECTED_EMBEDDING_PROVIDER\"\nLEGACY_EMBEDDING_ROUTE_PREFIX = \"legacy:\"\nINDEXED_EMBEDDING_ROUTE_PREFIX = \"space:\"\n\n\n@dataclass(frozen=True)\nclass EmbeddingSpace:\n \"\"\"One indexed vector space and the model route needed to query it.\"\"\"\n\n space_id: str\n route_model: str\n field_identity: str\n legacy: bool = False\n\n\ndef build_embedding_space_aggregation(\n *,\n size: int,\n qualified_after: dict[str, Any] | None = None,\n legacy_after: dict[str, Any] | None = None,\n include_qualified: bool = True,\n include_legacy: bool = True,\n) -> dict[str, Any]:\n \"\"\"Build pageable discovery aggregations for exact and legacy vector spaces.\"\"\"\n aggregations: dict[str, Any] = {}\n if include_qualified:\n composite: dict[str, Any] = {\n \"size\": size,\n \"sources\": [{\"space_id\": {\"terms\": {\"field\": \"embedding_space_id\"}}}],\n }\n if qualified_after:\n composite[\"after\"] = qualified_after\n aggregations[\"embedding_spaces\"] = {\"composite\": composite}\n\n if include_legacy:\n composite = {\n \"size\": size,\n \"sources\": [{\"model\": {\"terms\": {\"field\": \"embedding_model\"}}}],\n }\n if legacy_after:\n composite[\"after\"] = legacy_after\n aggregations[\"legacy_embedding_models\"] = {\"composite\": composite}\n return aggregations\n\n\ndef embedding_space_after_keys(\n result: dict[str, Any],\n) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:\n \"\"\"Return composite pagination cursors for exact and legacy aggregations.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_after = aggregations.get(\"embedding_spaces\", {}).get(\"after_key\")\n legacy_after = aggregations.get(\"legacy_embedding_models\", {}).get(\"after_key\")\n return qualified_after, legacy_after\n\n\ndef embedding_spaces_from_aggregation(result: dict[str, Any]) -> list[EmbeddingSpace]:\n \"\"\"Convert an OpenSearch aggregation response into stable retrieval identities.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_buckets = aggregations.get(\"embedding_spaces\", {}).get(\"buckets\", [])\n legacy_buckets = aggregations.get(\"legacy_embedding_models\", {}).get(\"buckets\", [])\n\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n for bucket in qualified_buckets:\n key = bucket.get(\"key\")\n raw_space_id = key.get(\"space_id\") if isinstance(key, dict) else key\n space_id = str(raw_space_id or \"\").strip()\n if not space_id or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=f\"{INDEXED_EMBEDDING_ROUTE_PREFIX}{space_id}\",\n field_identity=space_id,\n )\n )\n\n for bucket in legacy_buckets:\n key = bucket.get(\"key\")\n raw_model = key.get(\"model\") if isinstance(key, dict) else key\n model = str(raw_model or \"\").strip()\n space_id = f\"{LEGACY_EMBEDDING_ROUTE_PREFIX}{model}\"\n if not model or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=space_id,\n field_identity=model,\n legacy=True,\n )\n )\n return spaces\n\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {\n h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None\n }\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\nUNTRUSTED_CHUNK_FENCE_START = \"<<<UNTRUSTED_DOC_CHUNK>>>\"\nUNTRUSTED_CHUNK_FENCE_END = \"<<<END_UNTRUSTED_DOC_CHUNK>>>\"\n\n\ndef fence_untrusted_text(text: str) -> str:\n \"\"\"Wrap retrieved document text in delimiters marking it as untrusted data.\n\n Retrieved chunk text can contain attacker-controlled natural-language\n instructions (indirect prompt injection, VULN-13906). Fencing it lets the\n system prompt tell the LLM to treat anything between these markers as\n data only, never as instructions or tool directives.\n\n Any literal fence markers already present in `text` are escaped first, so\n a poisoned chunk can't embed a fake end-of-fence marker to break out of\n the untrusted section and have its continuation misread as trusted.\n \"\"\"\n if not text:\n return text\n escaped = text.replace(UNTRUSTED_CHUNK_FENCE_START, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_START).replace(\n UNTRUSTED_CHUNK_FENCE_END, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_END\n )\n return f\"{UNTRUSTED_CHUNK_FENCE_START}\\n{escaped}\\n{UNTRUSTED_CHUNK_FENCE_END}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"embedding_provider_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"embedding_provider_name\",\n display_name=\"Embedding Provider\",\n value=SELECTED_EMBEDDING_PROVIDER_VAR,\n info=(\n \"Exact OpenRAG provider key used for new ingestion metadata and vector fields. \"\n \"Bound to SELECTED_EMBEDDING_PROVIDER in managed OpenRAG flows.\"\n ),\n load_from_db=True,\n advanced=True,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"OPENSEARCH_PASSWORD\",\n load_from_db=True,\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False,\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n SecretStrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n if \"text\" in source:\n source[\"text\"] = fence_untrusted_text(source[\"text\"])\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n\n logger.debug(\n \"[OpenRAG callback config] configured for ingest_run_id=%s (url_len=%d, token_len=%d)\",\n ingest_run_id,\n len(url),\n len(token),\n )\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n logger.debug(\n f\"[OpenRAG ingest POST] total_chunks={len(requests)} batch_size={batch_size} timeout_s={timeout}\"\n )\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.debug(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n logger.debug(\n f\"[OpenRAG ingest POST resp] batch={batch_number} status={response.status_code}\"\n )\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code})\"\n )\n logger.warning(msg)\n raise RuntimeError(msg)\n\n logger.debug(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n embedding_provider: str | None = None,\n embedding_space_id: str | None = None,\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n embedding_provider: Exact provider key, when known\n embedding_space_id: Provider-qualified vector-space identity\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if embedding_provider:\n request[\"embedding_provider\"] = embedding_provider\n if embedding_space_id:\n request[\"embedding_space_id\"] = embedding_space_id\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", url),\n (\"openrag_ingest_token\", token),\n (\"openrag_ingest_run_id\", ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self._openrag_input_to_str(self.password) or \"\").strip()\n if pwd == \"OPENSEARCH_PASSWORD\" or not pwd:\n from config.settings import get_opensearch_password\n\n pwd = get_opensearch_password() or \"\"\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n embedding_provider = self._openrag_input_to_str(\n getattr(self, \"embedding_provider_name\", \"\")\n ).lower()\n if embedding_provider == SELECTED_EMBEDDING_PROVIDER_VAR.lower():\n embedding_provider = \"\"\n embedding_space_id = (\n f\"{embedding_provider}:{embedding_model}\" if embedding_provider else None\n )\n dynamic_field_name = get_embedding_field_name(embedding_space_id or embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n embedding_provider=embedding_provider or None,\n embedding_space_id=embedding_space_id,\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(\n self, client: OpenSearch, filter_clauses: list[dict] | None = None\n ) -> list[EmbeddingSpace]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n result: dict[str, Any] = {}\n\n # Composite aggregations must be top-level. Exact and legacy\n # spaces use different document filters, so paginate them in two\n # independent passes rather than nesting composite under filter.\n for legacy in (False, True):\n after = None\n while True:\n agg_query = {\n \"size\": 0,\n \"aggs\": build_embedding_space_aggregation(\n size=EMBEDDING_SPACE_PAGE_SIZE,\n qualified_after=None if legacy else after,\n legacy_after=after if legacy else None,\n include_qualified=not legacy,\n include_legacy=legacy,\n ),\n }\n bool_query: dict[str, Any] = {}\n if filter_clauses:\n bool_query[\"filter\"] = filter_clauses\n if legacy:\n bool_query[\"must_not\"] = [{\"exists\": {\"field\": \"embedding_space_id\"}}]\n if bool_query:\n agg_query[\"query\"] = {\"bool\": bool_query}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n for space in embedding_spaces_from_aggregation(result):\n if space.space_id not in seen:\n seen.add(space.space_id)\n spaces.append(space)\n\n qualified_after, legacy_after = embedding_space_after_keys(result)\n next_after = legacy_after if legacy else qualified_after\n if not next_after or next_after == after:\n break\n after = next_after\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding spaces in corpus: {[space.space_id for space in spaces]}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not spaces:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Provider provenance cannot be inferred safely after discovery\n # fails. Keyword search remains available without guessing.\n return []\n else:\n return spaces\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_spaces = self._detect_available_models(client, filter_clauses)\n\n if not available_spaces:\n logger.warning(\"No embedding spaces found in index; using keyword-only search\")\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.warning(\n \"No valid embedding adapter is available; continuing with keyword-only search.\"\n )\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_spaces)} spaces in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(\n f\"[SEARCH] Embedding spaces detected in index: \"\n f\"{[space.space_id for space in available_spaces]}\"\n )\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for space in available_spaces:\n model_name = space.space_id\n try:\n emb_obj = None\n\n # The OpenRAG proxy adapter can create an immutable child for\n # any provider-qualified or legacy route. Resolve through it\n # first so a legacy OpenAI vector is never queried through the\n # currently selected Azure/Watsonx provider by accident.\n for candidate in embeddings_list:\n resolver = getattr(candidate, \"for_model\", None)\n if callable(resolver):\n emb_obj = resolver(space.route_model)\n break\n\n # Backward compatibility for custom flows that still wire one\n # concrete embedding adapter per model.\n if emb_obj is None:\n for identifier in (space.space_id, space.field_identity):\n if identifier in embedding_by_model:\n emb_obj = embedding_by_model[identifier]\n break\n\n if emb_obj is not None:\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[space.field_identity] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except Exception as e: # provider clients expose different exception families\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n self.log(\n \"[WARN] No indexed embedding space could be resolved; \"\n \"continuing with keyword-only search\"\n )\n\n index_properties = self._get_index_properties(client)\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Continuing with keyword-only search.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index; using keyword search.\"\n )\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n keyword_query = {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3 if knn_queries_with_candidates else 1.0,\n }\n }\n should_queries: list[dict[str, Any]] = [keyword_query]\n if knn_queries_with_candidates:\n should_queries.insert(\n 0,\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0,\n \"boost\": 0.7,\n \"queries\": knn_queries_with_candidates,\n }\n },\n )\n\n # Keyword search intentionally has no embedding-field existence filter:\n # documents in an unresolved legacy space must remain discoverable.\n body = {\n \"query\": {\n \"bool\": {\n \"should\": should_queries,\n \"minimum_should_match\": 1,\n \"filter\": filter_clauses,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"embedding_provider\",\n \"embedding_space_id\",\n \"parser\",\n \"chunk_size\",\n \"chunk_overlap\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing hybrid search with {len(knn_queries_with_candidates)} resolved embedding spaces: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if knn_queries_with_candidates and use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif knn_queries_with_candidates and (\n \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered)\n ):\n logger.warning(\n \"KNN search failed for an indexed embedding field; retrying keyword-only\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"should\"] = [keyword_query]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"spaces_in_index={[space.space_id for space in available_spaces]}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": fence_untrusted_text(hit[\"_source\"].get(\"text\", \"\")),\n \"metadata\": {\n **{k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"chunk_id\": hit.get(\"_id\"),\n \"id\": hit.get(\"_id\"),\n \"score\": hit.get(\"_score\"),\n },\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as a list of Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in a format that downstream Parser\n components and tool wrappers can consume directly without DataFrame string truncation.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n List containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return raw_list\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"+ "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport unicodedata\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom dataclasses import dataclass\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\nEMBEDDING_SPACE_PAGE_SIZE = 100\nSELECTED_EMBEDDING_PROVIDER_VAR = \"SELECTED_EMBEDDING_PROVIDER\"\nLEGACY_EMBEDDING_ROUTE_PREFIX = \"legacy:\"\nINDEXED_EMBEDDING_ROUTE_PREFIX = \"space:\"\n\n# Some PDF fonts map common kanji (日, 月, 木, 金, 土, ...) to the look-alike\n# Kangxi Radical code points in their /ToUnicode CMap, and Docling copies them\n# through unchanged. Map just those blocks back to CJK Unified Ideographs;\n# full NFKC would also rewrite full-width digits, ㈱, ①, half-width katakana.\n# Mirrors normalize_cjk_radicals in src/utils/document_processing.py.\nCJK_RADICAL_TABLE = {\n cp: unicodedata.normalize(\"NFKC\", chr(cp))\n for cp in range(0x2E80, 0x2FE0) # CJK Radicals Supplement + Kangxi Radicals\n if unicodedata.normalize(\"NFKC\", chr(cp)) != chr(cp)\n}\n\n\n@dataclass(frozen=True)\nclass EmbeddingSpace:\n \"\"\"One indexed vector space and the model route needed to query it.\"\"\"\n\n space_id: str\n route_model: str\n field_identity: str\n legacy: bool = False\n\n\ndef build_embedding_space_aggregation(\n *,\n size: int,\n qualified_after: dict[str, Any] | None = None,\n legacy_after: dict[str, Any] | None = None,\n include_qualified: bool = True,\n include_legacy: bool = True,\n) -> dict[str, Any]:\n \"\"\"Build pageable discovery aggregations for exact and legacy vector spaces.\"\"\"\n aggregations: dict[str, Any] = {}\n if include_qualified:\n composite: dict[str, Any] = {\n \"size\": size,\n \"sources\": [{\"space_id\": {\"terms\": {\"field\": \"embedding_space_id\"}}}],\n }\n if qualified_after:\n composite[\"after\"] = qualified_after\n aggregations[\"embedding_spaces\"] = {\"composite\": composite}\n\n if include_legacy:\n composite = {\n \"size\": size,\n \"sources\": [{\"model\": {\"terms\": {\"field\": \"embedding_model\"}}}],\n }\n if legacy_after:\n composite[\"after\"] = legacy_after\n aggregations[\"legacy_embedding_models\"] = {\"composite\": composite}\n return aggregations\n\n\ndef embedding_space_after_keys(\n result: dict[str, Any],\n) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:\n \"\"\"Return composite pagination cursors for exact and legacy aggregations.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_after = aggregations.get(\"embedding_spaces\", {}).get(\"after_key\")\n legacy_after = aggregations.get(\"legacy_embedding_models\", {}).get(\"after_key\")\n return qualified_after, legacy_after\n\n\ndef embedding_spaces_from_aggregation(result: dict[str, Any]) -> list[EmbeddingSpace]:\n \"\"\"Convert an OpenSearch aggregation response into stable retrieval identities.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_buckets = aggregations.get(\"embedding_spaces\", {}).get(\"buckets\", [])\n legacy_buckets = aggregations.get(\"legacy_embedding_models\", {}).get(\"buckets\", [])\n\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n for bucket in qualified_buckets:\n key = bucket.get(\"key\")\n raw_space_id = key.get(\"space_id\") if isinstance(key, dict) else key\n space_id = str(raw_space_id or \"\").strip()\n if not space_id or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=f\"{INDEXED_EMBEDDING_ROUTE_PREFIX}{space_id}\",\n field_identity=space_id,\n )\n )\n\n for bucket in legacy_buckets:\n key = bucket.get(\"key\")\n raw_model = key.get(\"model\") if isinstance(key, dict) else key\n model = str(raw_model or \"\").strip()\n space_id = f\"{LEGACY_EMBEDDING_ROUTE_PREFIX}{model}\"\n if not model or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=space_id,\n field_identity=model,\n legacy=True,\n )\n )\n return spaces\n\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {\n h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None\n }\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\nUNTRUSTED_CHUNK_FENCE_START = \"<<<UNTRUSTED_DOC_CHUNK>>>\"\nUNTRUSTED_CHUNK_FENCE_END = \"<<<END_UNTRUSTED_DOC_CHUNK>>>\"\n\n\ndef fence_untrusted_text(text: str) -> str:\n \"\"\"Wrap retrieved document text in delimiters marking it as untrusted data.\n\n Retrieved chunk text can contain attacker-controlled natural-language\n instructions (indirect prompt injection, VULN-13906). Fencing it lets the\n system prompt tell the LLM to treat anything between these markers as\n data only, never as instructions or tool directives.\n\n Any literal fence markers already present in `text` are escaped first, so\n a poisoned chunk can't embed a fake end-of-fence marker to break out of\n the untrusted section and have its continuation misread as trusted.\n \"\"\"\n if not text:\n return text\n escaped = text.replace(UNTRUSTED_CHUNK_FENCE_START, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_START).replace(\n UNTRUSTED_CHUNK_FENCE_END, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_END\n )\n return f\"{UNTRUSTED_CHUNK_FENCE_START}\\n{escaped}\\n{UNTRUSTED_CHUNK_FENCE_END}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"embedding_provider_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"embedding_provider_name\",\n display_name=\"Embedding Provider\",\n value=SELECTED_EMBEDDING_PROVIDER_VAR,\n info=(\n \"Exact OpenRAG provider key used for new ingestion metadata and vector fields. \"\n \"Bound to SELECTED_EMBEDDING_PROVIDER in managed OpenRAG flows.\"\n ),\n load_from_db=True,\n advanced=True,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"OPENSEARCH_PASSWORD\",\n load_from_db=True,\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False,\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n SecretStrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n if \"text\" in source:\n source[\"text\"] = fence_untrusted_text(source[\"text\"])\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n\n logger.debug(\n \"[OpenRAG callback config] configured for ingest_run_id=%s (url_len=%d, token_len=%d)\",\n ingest_run_id,\n len(url),\n len(token),\n )\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n logger.debug(\n f\"[OpenRAG ingest POST] total_chunks={len(requests)} batch_size={batch_size} timeout_s={timeout}\"\n )\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.debug(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n logger.debug(\n f\"[OpenRAG ingest POST resp] batch={batch_number} status={response.status_code}\"\n )\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code})\"\n )\n logger.warning(msg)\n raise RuntimeError(msg)\n\n logger.debug(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n embedding_provider: str | None = None,\n embedding_space_id: str | None = None,\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n embedding_provider: Exact provider key, when known\n embedding_space_id: Provider-qualified vector-space identity\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if embedding_provider:\n request[\"embedding_provider\"] = embedding_provider\n if embedding_space_id:\n request[\"embedding_space_id\"] = embedding_space_id\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", url),\n (\"openrag_ingest_token\", token),\n (\"openrag_ingest_run_id\", ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self._openrag_input_to_str(self.password) or \"\").strip()\n if pwd == \"OPENSEARCH_PASSWORD\" or not pwd:\n from config.settings import get_opensearch_password\n\n pwd = get_opensearch_password() or \"\"\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n embedding_provider = self._openrag_input_to_str(\n getattr(self, \"embedding_provider_name\", \"\")\n ).lower()\n if embedding_provider == SELECTED_EMBEDDING_PROVIDER_VAR.lower():\n embedding_provider = \"\"\n embedding_space_id = (\n f\"{embedding_provider}:{embedding_model}\" if embedding_provider else None\n )\n dynamic_field_name = get_embedding_field_name(embedding_space_id or embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n if isinstance(text, str):\n text = text.translate(CJK_RADICAL_TABLE)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n embedding_provider=embedding_provider or None,\n embedding_space_id=embedding_space_id,\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(\n self, client: OpenSearch, filter_clauses: list[dict] | None = None\n ) -> list[EmbeddingSpace]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n result: dict[str, Any] = {}\n\n # Composite aggregations must be top-level. Exact and legacy\n # spaces use different document filters, so paginate them in two\n # independent passes rather than nesting composite under filter.\n for legacy in (False, True):\n after = None\n while True:\n agg_query = {\n \"size\": 0,\n \"aggs\": build_embedding_space_aggregation(\n size=EMBEDDING_SPACE_PAGE_SIZE,\n qualified_after=None if legacy else after,\n legacy_after=after if legacy else None,\n include_qualified=not legacy,\n include_legacy=legacy,\n ),\n }\n bool_query: dict[str, Any] = {}\n if filter_clauses:\n bool_query[\"filter\"] = filter_clauses\n if legacy:\n bool_query[\"must_not\"] = [{\"exists\": {\"field\": \"embedding_space_id\"}}]\n if bool_query:\n agg_query[\"query\"] = {\"bool\": bool_query}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n for space in embedding_spaces_from_aggregation(result):\n if space.space_id not in seen:\n seen.add(space.space_id)\n spaces.append(space)\n\n qualified_after, legacy_after = embedding_space_after_keys(result)\n next_after = legacy_after if legacy else qualified_after\n if not next_after or next_after == after:\n break\n after = next_after\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding spaces in corpus: {[space.space_id for space in spaces]}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not spaces:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Provider provenance cannot be inferred safely after discovery\n # fails. Keyword search remains available without guessing.\n return []\n else:\n return spaces\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_spaces = self._detect_available_models(client, filter_clauses)\n\n if not available_spaces:\n logger.warning(\"No embedding spaces found in index; using keyword-only search\")\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.warning(\n \"No valid embedding adapter is available; continuing with keyword-only search.\"\n )\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_spaces)} spaces in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(\n f\"[SEARCH] Embedding spaces detected in index: \"\n f\"{[space.space_id for space in available_spaces]}\"\n )\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for space in available_spaces:\n model_name = space.space_id\n try:\n emb_obj = None\n\n # The OpenRAG proxy adapter can create an immutable child for\n # any provider-qualified or legacy route. Resolve through it\n # first so a legacy OpenAI vector is never queried through the\n # currently selected Azure/Watsonx provider by accident.\n for candidate in embeddings_list:\n resolver = getattr(candidate, \"for_model\", None)\n if callable(resolver):\n emb_obj = resolver(space.route_model)\n break\n\n # Backward compatibility for custom flows that still wire one\n # concrete embedding adapter per model.\n if emb_obj is None:\n for identifier in (space.space_id, space.field_identity):\n if identifier in embedding_by_model:\n emb_obj = embedding_by_model[identifier]\n break\n\n if emb_obj is not None:\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[space.field_identity] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except Exception as e: # provider clients expose different exception families\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n self.log(\n \"[WARN] No indexed embedding space could be resolved; \"\n \"continuing with keyword-only search\"\n )\n\n index_properties = self._get_index_properties(client)\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Continuing with keyword-only search.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index; using keyword search.\"\n )\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n keyword_query = {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3 if knn_queries_with_candidates else 1.0,\n }\n }\n should_queries: list[dict[str, Any]] = [keyword_query]\n if knn_queries_with_candidates:\n should_queries.insert(\n 0,\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0,\n \"boost\": 0.7,\n \"queries\": knn_queries_with_candidates,\n }\n },\n )\n\n # Keyword search intentionally has no embedding-field existence filter:\n # documents in an unresolved legacy space must remain discoverable.\n body = {\n \"query\": {\n \"bool\": {\n \"should\": should_queries,\n \"minimum_should_match\": 1,\n \"filter\": filter_clauses,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"embedding_provider\",\n \"embedding_space_id\",\n \"parser\",\n \"chunk_size\",\n \"chunk_overlap\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing hybrid search with {len(knn_queries_with_candidates)} resolved embedding spaces: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if knn_queries_with_candidates and use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif knn_queries_with_candidates and (\n \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered)\n ):\n logger.warning(\n \"KNN search failed for an indexed embedding field; retrying keyword-only\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"should\"] = [keyword_query]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"spaces_in_index={[space.space_id for space in available_spaces]}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": fence_untrusted_text(hit[\"_source\"].get(\"text\", \"\")),\n \"metadata\": {\n **{k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"chunk_id\": hit.get(\"_id\"),\n \"id\": hit.get(\"_id\"),\n \"score\": hit.get(\"_score\"),\n },\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as a list of Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in a format that downstream Parser\n components and tool wrappers can consume directly without DataFrame string truncation.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n List containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return raw_list\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"},"docs_metadata": {"_input_type": "TableInput",diff --git a/flows/openrag_url_mcp.json b/flows/openrag_url_mcp.jsonindex cf65f1863..78d8d60ee 100644--- a/flows/openrag_url_mcp.json+++ b/flows/openrag_url_mcp.json@@ -5925,7 +5925,7 @@"last_updated": "2026-08-18T22:49:22.128Z","legacy": false,"metadata": {- "code_hash": "a3727d7c66eb",+ "code_hash": "1a2a5b8d0ca7","dependencies": {"dependencies": [{@@ -6086,7 +6086,7 @@"show": true,"title_case": false,"type": "code",- "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom dataclasses import dataclass\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\nEMBEDDING_SPACE_PAGE_SIZE = 100\nSELECTED_EMBEDDING_PROVIDER_VAR = \"SELECTED_EMBEDDING_PROVIDER\"\nLEGACY_EMBEDDING_ROUTE_PREFIX = \"legacy:\"\nINDEXED_EMBEDDING_ROUTE_PREFIX = \"space:\"\n\n\n@dataclass(frozen=True)\nclass EmbeddingSpace:\n \"\"\"One indexed vector space and the model route needed to query it.\"\"\"\n\n space_id: str\n route_model: str\n field_identity: str\n legacy: bool = False\n\n\ndef build_embedding_space_aggregation(\n *,\n size: int,\n qualified_after: dict[str, Any] | None = None,\n legacy_after: dict[str, Any] | None = None,\n include_qualified: bool = True,\n include_legacy: bool = True,\n) -> dict[str, Any]:\n \"\"\"Build pageable discovery aggregations for exact and legacy vector spaces.\"\"\"\n aggregations: dict[str, Any] = {}\n if include_qualified:\n composite: dict[str, Any] = {\n \"size\": size,\n \"sources\": [{\"space_id\": {\"terms\": {\"field\": \"embedding_space_id\"}}}],\n }\n if qualified_after:\n composite[\"after\"] = qualified_after\n aggregations[\"embedding_spaces\"] = {\"composite\": composite}\n\n if include_legacy:\n composite = {\n \"size\": size,\n \"sources\": [{\"model\": {\"terms\": {\"field\": \"embedding_model\"}}}],\n }\n if legacy_after:\n composite[\"after\"] = legacy_after\n aggregations[\"legacy_embedding_models\"] = {\"composite\": composite}\n return aggregations\n\n\ndef embedding_space_after_keys(\n result: dict[str, Any],\n) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:\n \"\"\"Return composite pagination cursors for exact and legacy aggregations.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_after = aggregations.get(\"embedding_spaces\", {}).get(\"after_key\")\n legacy_after = aggregations.get(\"legacy_embedding_models\", {}).get(\"after_key\")\n return qualified_after, legacy_after\n\n\ndef embedding_spaces_from_aggregation(result: dict[str, Any]) -> list[EmbeddingSpace]:\n \"\"\"Convert an OpenSearch aggregation response into stable retrieval identities.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_buckets = aggregations.get(\"embedding_spaces\", {}).get(\"buckets\", [])\n legacy_buckets = aggregations.get(\"legacy_embedding_models\", {}).get(\"buckets\", [])\n\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n for bucket in qualified_buckets:\n key = bucket.get(\"key\")\n raw_space_id = key.get(\"space_id\") if isinstance(key, dict) else key\n space_id = str(raw_space_id or \"\").strip()\n if not space_id or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=f\"{INDEXED_EMBEDDING_ROUTE_PREFIX}{space_id}\",\n field_identity=space_id,\n )\n )\n\n for bucket in legacy_buckets:\n key = bucket.get(\"key\")\n raw_model = key.get(\"model\") if isinstance(key, dict) else key\n model = str(raw_model or \"\").strip()\n space_id = f\"{LEGACY_EMBEDDING_ROUTE_PREFIX}{model}\"\n if not model or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=space_id,\n field_identity=model,\n legacy=True,\n )\n )\n return spaces\n\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {\n h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None\n }\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\nUNTRUSTED_CHUNK_FENCE_START = \"<<<UNTRUSTED_DOC_CHUNK>>>\"\nUNTRUSTED_CHUNK_FENCE_END = \"<<<END_UNTRUSTED_DOC_CHUNK>>>\"\n\n\ndef fence_untrusted_text(text: str) -> str:\n \"\"\"Wrap retrieved document text in delimiters marking it as untrusted data.\n\n Retrieved chunk text can contain attacker-controlled natural-language\n instructions (indirect prompt injection, VULN-13906). Fencing it lets the\n system prompt tell the LLM to treat anything between these markers as\n data only, never as instructions or tool directives.\n\n Any literal fence markers already present in `text` are escaped first, so\n a poisoned chunk can't embed a fake end-of-fence marker to break out of\n the untrusted section and have its continuation misread as trusted.\n \"\"\"\n if not text:\n return text\n escaped = text.replace(UNTRUSTED_CHUNK_FENCE_START, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_START).replace(\n UNTRUSTED_CHUNK_FENCE_END, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_END\n )\n return f\"{UNTRUSTED_CHUNK_FENCE_START}\\n{escaped}\\n{UNTRUSTED_CHUNK_FENCE_END}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"embedding_provider_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"embedding_provider_name\",\n display_name=\"Embedding Provider\",\n value=SELECTED_EMBEDDING_PROVIDER_VAR,\n info=(\n \"Exact OpenRAG provider key used for new ingestion metadata and vector fields. \"\n \"Bound to SELECTED_EMBEDDING_PROVIDER in managed OpenRAG flows.\"\n ),\n load_from_db=True,\n advanced=True,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"OPENSEARCH_PASSWORD\",\n load_from_db=True,\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False,\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n SecretStrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n if \"text\" in source:\n source[\"text\"] = fence_untrusted_text(source[\"text\"])\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n\n logger.debug(\n \"[OpenRAG callback config] configured for ingest_run_id=%s (url_len=%d, token_len=%d)\",\n ingest_run_id,\n len(url),\n len(token),\n )\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n logger.debug(\n f\"[OpenRAG ingest POST] total_chunks={len(requests)} batch_size={batch_size} timeout_s={timeout}\"\n )\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.debug(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n logger.debug(\n f\"[OpenRAG ingest POST resp] batch={batch_number} status={response.status_code}\"\n )\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code})\"\n )\n logger.warning(msg)\n raise RuntimeError(msg)\n\n logger.debug(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n embedding_provider: str | None = None,\n embedding_space_id: str | None = None,\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n embedding_provider: Exact provider key, when known\n embedding_space_id: Provider-qualified vector-space identity\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if embedding_provider:\n request[\"embedding_provider\"] = embedding_provider\n if embedding_space_id:\n request[\"embedding_space_id\"] = embedding_space_id\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", url),\n (\"openrag_ingest_token\", token),\n (\"openrag_ingest_run_id\", ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self._openrag_input_to_str(self.password) or \"\").strip()\n if pwd == \"OPENSEARCH_PASSWORD\" or not pwd:\n from config.settings import get_opensearch_password\n\n pwd = get_opensearch_password() or \"\"\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n embedding_provider = self._openrag_input_to_str(\n getattr(self, \"embedding_provider_name\", \"\")\n ).lower()\n if embedding_provider == SELECTED_EMBEDDING_PROVIDER_VAR.lower():\n embedding_provider = \"\"\n embedding_space_id = (\n f\"{embedding_provider}:{embedding_model}\" if embedding_provider else None\n )\n dynamic_field_name = get_embedding_field_name(embedding_space_id or embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n embedding_provider=embedding_provider or None,\n embedding_space_id=embedding_space_id,\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(\n self, client: OpenSearch, filter_clauses: list[dict] | None = None\n ) -> list[EmbeddingSpace]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n result: dict[str, Any] = {}\n\n # Composite aggregations must be top-level. Exact and legacy\n # spaces use different document filters, so paginate them in two\n # independent passes rather than nesting composite under filter.\n for legacy in (False, True):\n after = None\n while True:\n agg_query = {\n \"size\": 0,\n \"aggs\": build_embedding_space_aggregation(\n size=EMBEDDING_SPACE_PAGE_SIZE,\n qualified_after=None if legacy else after,\n legacy_after=after if legacy else None,\n include_qualified=not legacy,\n include_legacy=legacy,\n ),\n }\n bool_query: dict[str, Any] = {}\n if filter_clauses:\n bool_query[\"filter\"] = filter_clauses\n if legacy:\n bool_query[\"must_not\"] = [{\"exists\": {\"field\": \"embedding_space_id\"}}]\n if bool_query:\n agg_query[\"query\"] = {\"bool\": bool_query}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n for space in embedding_spaces_from_aggregation(result):\n if space.space_id not in seen:\n seen.add(space.space_id)\n spaces.append(space)\n\n qualified_after, legacy_after = embedding_space_after_keys(result)\n next_after = legacy_after if legacy else qualified_after\n if not next_after or next_after == after:\n break\n after = next_after\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding spaces in corpus: {[space.space_id for space in spaces]}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not spaces:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Provider provenance cannot be inferred safely after discovery\n # fails. Keyword search remains available without guessing.\n return []\n else:\n return spaces\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_spaces = self._detect_available_models(client, filter_clauses)\n\n if not available_spaces:\n logger.warning(\"No embedding spaces found in index; using keyword-only search\")\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.warning(\n \"No valid embedding adapter is available; continuing with keyword-only search.\"\n )\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_spaces)} spaces in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(\n f\"[SEARCH] Embedding spaces detected in index: \"\n f\"{[space.space_id for space in available_spaces]}\"\n )\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for space in available_spaces:\n model_name = space.space_id\n try:\n emb_obj = None\n\n # The OpenRAG proxy adapter can create an immutable child for\n # any provider-qualified or legacy route. Resolve through it\n # first so a legacy OpenAI vector is never queried through the\n # currently selected Azure/Watsonx provider by accident.\n for candidate in embeddings_list:\n resolver = getattr(candidate, \"for_model\", None)\n if callable(resolver):\n emb_obj = resolver(space.route_model)\n break\n\n # Backward compatibility for custom flows that still wire one\n # concrete embedding adapter per model.\n if emb_obj is None:\n for identifier in (space.space_id, space.field_identity):\n if identifier in embedding_by_model:\n emb_obj = embedding_by_model[identifier]\n break\n\n if emb_obj is not None:\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[space.field_identity] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except Exception as e: # provider clients expose different exception families\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n self.log(\n \"[WARN] No indexed embedding space could be resolved; \"\n \"continuing with keyword-only search\"\n )\n\n index_properties = self._get_index_properties(client)\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Continuing with keyword-only search.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index; using keyword search.\"\n )\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n keyword_query = {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3 if knn_queries_with_candidates else 1.0,\n }\n }\n should_queries: list[dict[str, Any]] = [keyword_query]\n if knn_queries_with_candidates:\n should_queries.insert(\n 0,\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0,\n \"boost\": 0.7,\n \"queries\": knn_queries_with_candidates,\n }\n },\n )\n\n # Keyword search intentionally has no embedding-field existence filter:\n # documents in an unresolved legacy space must remain discoverable.\n body = {\n \"query\": {\n \"bool\": {\n \"should\": should_queries,\n \"minimum_should_match\": 1,\n \"filter\": filter_clauses,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"embedding_provider\",\n \"embedding_space_id\",\n \"parser\",\n \"chunk_size\",\n \"chunk_overlap\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing hybrid search with {len(knn_queries_with_candidates)} resolved embedding spaces: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if knn_queries_with_candidates and use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif knn_queries_with_candidates and (\n \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered)\n ):\n logger.warning(\n \"KNN search failed for an indexed embedding field; retrying keyword-only\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"should\"] = [keyword_query]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"spaces_in_index={[space.space_id for space in available_spaces]}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": fence_untrusted_text(hit[\"_source\"].get(\"text\", \"\")),\n \"metadata\": {\n **{k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"chunk_id\": hit.get(\"_id\"),\n \"id\": hit.get(\"_id\"),\n \"score\": hit.get(\"_score\"),\n },\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as a list of Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in a format that downstream Parser\n components and tool wrappers can consume directly without DataFrame string truncation.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n List containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return raw_list\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"+ "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport unicodedata\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom dataclasses import dataclass\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\nEMBEDDING_SPACE_PAGE_SIZE = 100\nSELECTED_EMBEDDING_PROVIDER_VAR = \"SELECTED_EMBEDDING_PROVIDER\"\nLEGACY_EMBEDDING_ROUTE_PREFIX = \"legacy:\"\nINDEXED_EMBEDDING_ROUTE_PREFIX = \"space:\"\n\n# Some PDF fonts map common kanji (日, 月, 木, 金, 土, ...) to the look-alike\n# Kangxi Radical code points in their /ToUnicode CMap, and Docling copies them\n# through unchanged. Map just those blocks back to CJK Unified Ideographs;\n# full NFKC would also rewrite full-width digits, ㈱, ①, half-width katakana.\n# Mirrors normalize_cjk_radicals in src/utils/document_processing.py.\nCJK_RADICAL_TABLE = {\n cp: unicodedata.normalize(\"NFKC\", chr(cp))\n for cp in range(0x2E80, 0x2FE0) # CJK Radicals Supplement + Kangxi Radicals\n if unicodedata.normalize(\"NFKC\", chr(cp)) != chr(cp)\n}\n\n\n@dataclass(frozen=True)\nclass EmbeddingSpace:\n \"\"\"One indexed vector space and the model route needed to query it.\"\"\"\n\n space_id: str\n route_model: str\n field_identity: str\n legacy: bool = False\n\n\ndef build_embedding_space_aggregation(\n *,\n size: int,\n qualified_after: dict[str, Any] | None = None,\n legacy_after: dict[str, Any] | None = None,\n include_qualified: bool = True,\n include_legacy: bool = True,\n) -> dict[str, Any]:\n \"\"\"Build pageable discovery aggregations for exact and legacy vector spaces.\"\"\"\n aggregations: dict[str, Any] = {}\n if include_qualified:\n composite: dict[str, Any] = {\n \"size\": size,\n \"sources\": [{\"space_id\": {\"terms\": {\"field\": \"embedding_space_id\"}}}],\n }\n if qualified_after:\n composite[\"after\"] = qualified_after\n aggregations[\"embedding_spaces\"] = {\"composite\": composite}\n\n if include_legacy:\n composite = {\n \"size\": size,\n \"sources\": [{\"model\": {\"terms\": {\"field\": \"embedding_model\"}}}],\n }\n if legacy_after:\n composite[\"after\"] = legacy_after\n aggregations[\"legacy_embedding_models\"] = {\"composite\": composite}\n return aggregations\n\n\ndef embedding_space_after_keys(\n result: dict[str, Any],\n) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:\n \"\"\"Return composite pagination cursors for exact and legacy aggregations.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_after = aggregations.get(\"embedding_spaces\", {}).get(\"after_key\")\n legacy_after = aggregations.get(\"legacy_embedding_models\", {}).get(\"after_key\")\n return qualified_after, legacy_after\n\n\ndef embedding_spaces_from_aggregation(result: dict[str, Any]) -> list[EmbeddingSpace]:\n \"\"\"Convert an OpenSearch aggregation response into stable retrieval identities.\"\"\"\n aggregations = result.get(\"aggregations\", {})\n qualified_buckets = aggregations.get(\"embedding_spaces\", {}).get(\"buckets\", [])\n legacy_buckets = aggregations.get(\"legacy_embedding_models\", {}).get(\"buckets\", [])\n\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n for bucket in qualified_buckets:\n key = bucket.get(\"key\")\n raw_space_id = key.get(\"space_id\") if isinstance(key, dict) else key\n space_id = str(raw_space_id or \"\").strip()\n if not space_id or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=f\"{INDEXED_EMBEDDING_ROUTE_PREFIX}{space_id}\",\n field_identity=space_id,\n )\n )\n\n for bucket in legacy_buckets:\n key = bucket.get(\"key\")\n raw_model = key.get(\"model\") if isinstance(key, dict) else key\n model = str(raw_model or \"\").strip()\n space_id = f\"{LEGACY_EMBEDDING_ROUTE_PREFIX}{model}\"\n if not model or space_id in seen:\n continue\n seen.add(space_id)\n spaces.append(\n EmbeddingSpace(\n space_id=space_id,\n route_model=space_id,\n field_identity=model,\n legacy=True,\n )\n )\n return spaces\n\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {\n h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None\n }\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\nUNTRUSTED_CHUNK_FENCE_START = \"<<<UNTRUSTED_DOC_CHUNK>>>\"\nUNTRUSTED_CHUNK_FENCE_END = \"<<<END_UNTRUSTED_DOC_CHUNK>>>\"\n\n\ndef fence_untrusted_text(text: str) -> str:\n \"\"\"Wrap retrieved document text in delimiters marking it as untrusted data.\n\n Retrieved chunk text can contain attacker-controlled natural-language\n instructions (indirect prompt injection, VULN-13906). Fencing it lets the\n system prompt tell the LLM to treat anything between these markers as\n data only, never as instructions or tool directives.\n\n Any literal fence markers already present in `text` are escaped first, so\n a poisoned chunk can't embed a fake end-of-fence marker to break out of\n the untrusted section and have its continuation misread as trusted.\n \"\"\"\n if not text:\n return text\n escaped = text.replace(UNTRUSTED_CHUNK_FENCE_START, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_START).replace(\n UNTRUSTED_CHUNK_FENCE_END, \"\\\\\" + UNTRUSTED_CHUNK_FENCE_END\n )\n return f\"{UNTRUSTED_CHUNK_FENCE_START}\\n{escaped}\\n{UNTRUSTED_CHUNK_FENCE_END}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"embedding_provider_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"embedding_provider_name\",\n display_name=\"Embedding Provider\",\n value=SELECTED_EMBEDDING_PROVIDER_VAR,\n info=(\n \"Exact OpenRAG provider key used for new ingestion metadata and vector fields. \"\n \"Bound to SELECTED_EMBEDDING_PROVIDER in managed OpenRAG flows.\"\n ),\n load_from_db=True,\n advanced=True,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"OPENSEARCH_PASSWORD\",\n load_from_db=True,\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False,\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n SecretStrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n if \"text\" in source:\n source[\"text\"] = fence_untrusted_text(source[\"text\"])\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_provider\": {\"type\": \"keyword\"},\n \"embedding_space_id\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n\n logger.debug(\n \"[OpenRAG callback config] configured for ingest_run_id=%s (url_len=%d, token_len=%d)\",\n ingest_run_id,\n len(url),\n len(token),\n )\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n logger.debug(\n f\"[OpenRAG ingest POST] total_chunks={len(requests)} batch_size={batch_size} timeout_s={timeout}\"\n )\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.debug(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n logger.debug(\n f\"[OpenRAG ingest POST resp] batch={batch_number} status={response.status_code}\"\n )\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code})\"\n )\n logger.warning(msg)\n raise RuntimeError(msg)\n\n logger.debug(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n embedding_provider: str | None = None,\n embedding_space_id: str | None = None,\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n embedding_provider: Exact provider key, when known\n embedding_space_id: Provider-qualified vector-space identity\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if embedding_provider:\n request[\"embedding_provider\"] = embedding_provider\n if embedding_space_id:\n request[\"embedding_space_id\"] = embedding_space_id\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", url),\n (\"openrag_ingest_token\", token),\n (\"openrag_ingest_run_id\", ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self._openrag_input_to_str(self.password) or \"\").strip()\n if pwd == \"OPENSEARCH_PASSWORD\" or not pwd:\n from config.settings import get_opensearch_password\n\n pwd = get_opensearch_password() or \"\"\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n embedding_provider = self._openrag_input_to_str(\n getattr(self, \"embedding_provider_name\", \"\")\n ).lower()\n if embedding_provider == SELECTED_EMBEDDING_PROVIDER_VAR.lower():\n embedding_provider = \"\"\n embedding_space_id = (\n f\"{embedding_provider}:{embedding_model}\" if embedding_provider else None\n )\n dynamic_field_name = get_embedding_field_name(embedding_space_id or embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n if isinstance(text, str):\n text = text.translate(CJK_RADICAL_TABLE)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n embedding_provider=embedding_provider or None,\n embedding_space_id=embedding_space_id,\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(\n self, client: OpenSearch, filter_clauses: list[dict] | None = None\n ) -> list[EmbeddingSpace]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n spaces: list[EmbeddingSpace] = []\n seen: set[str] = set()\n result: dict[str, Any] = {}\n\n # Composite aggregations must be top-level. Exact and legacy\n # spaces use different document filters, so paginate them in two\n # independent passes rather than nesting composite under filter.\n for legacy in (False, True):\n after = None\n while True:\n agg_query = {\n \"size\": 0,\n \"aggs\": build_embedding_space_aggregation(\n size=EMBEDDING_SPACE_PAGE_SIZE,\n qualified_after=None if legacy else after,\n legacy_after=after if legacy else None,\n include_qualified=not legacy,\n include_legacy=legacy,\n ),\n }\n bool_query: dict[str, Any] = {}\n if filter_clauses:\n bool_query[\"filter\"] = filter_clauses\n if legacy:\n bool_query[\"must_not\"] = [{\"exists\": {\"field\": \"embedding_space_id\"}}]\n if bool_query:\n agg_query[\"query\"] = {\"bool\": bool_query}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n for space in embedding_spaces_from_aggregation(result):\n if space.space_id not in seen:\n seen.add(space.space_id)\n spaces.append(space)\n\n qualified_after, legacy_after = embedding_space_after_keys(result)\n next_after = legacy_after if legacy else qualified_after\n if not next_after or next_after == after:\n break\n after = next_after\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding spaces in corpus: {[space.space_id for space in spaces]}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not spaces:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Provider provenance cannot be inferred safely after discovery\n # fails. Keyword search remains available without guessing.\n return []\n else:\n return spaces\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_spaces = self._detect_available_models(client, filter_clauses)\n\n if not available_spaces:\n logger.warning(\"No embedding spaces found in index; using keyword-only search\")\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.warning(\n \"No valid embedding adapter is available; continuing with keyword-only search.\"\n )\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_spaces)} spaces in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(\n f\"[SEARCH] Embedding spaces detected in index: \"\n f\"{[space.space_id for space in available_spaces]}\"\n )\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for space in available_spaces:\n model_name = space.space_id\n try:\n emb_obj = None\n\n # The OpenRAG proxy adapter can create an immutable child for\n # any provider-qualified or legacy route. Resolve through it\n # first so a legacy OpenAI vector is never queried through the\n # currently selected Azure/Watsonx provider by accident.\n for candidate in embeddings_list:\n resolver = getattr(candidate, \"for_model\", None)\n if callable(resolver):\n emb_obj = resolver(space.route_model)\n break\n\n # Backward compatibility for custom flows that still wire one\n # concrete embedding adapter per model.\n if emb_obj is None:\n for identifier in (space.space_id, space.field_identity):\n if identifier in embedding_by_model:\n emb_obj = embedding_by_model[identifier]\n break\n\n if emb_obj is not None:\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[space.field_identity] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except Exception as e: # provider clients expose different exception families\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n self.log(\n \"[WARN] No indexed embedding space could be resolved; \"\n \"continuing with keyword-only search\"\n )\n\n index_properties = self._get_index_properties(client)\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Continuing with keyword-only search.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index; using keyword search.\"\n )\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n keyword_query = {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3 if knn_queries_with_candidates else 1.0,\n }\n }\n should_queries: list[dict[str, Any]] = [keyword_query]\n if knn_queries_with_candidates:\n should_queries.insert(\n 0,\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0,\n \"boost\": 0.7,\n \"queries\": knn_queries_with_candidates,\n }\n },\n )\n\n # Keyword search intentionally has no embedding-field existence filter:\n # documents in an unresolved legacy space must remain discoverable.\n body = {\n \"query\": {\n \"bool\": {\n \"should\": should_queries,\n \"minimum_should_match\": 1,\n \"filter\": filter_clauses,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"embedding_provider\",\n \"embedding_space_id\",\n \"parser\",\n \"chunk_size\",\n \"chunk_overlap\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing hybrid search with {len(knn_queries_with_candidates)} resolved embedding spaces: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if knn_queries_with_candidates and use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif knn_queries_with_candidates and (\n \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered)\n ):\n logger.warning(\n \"KNN search failed for an indexed embedding field; retrying keyword-only\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"should\"] = [keyword_query]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"spaces_in_index={[space.space_id for space in available_spaces]}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": fence_untrusted_text(hit[\"_source\"].get(\"text\", \"\")),\n \"metadata\": {\n **{k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"chunk_id\": hit.get(\"_id\"),\n \"id\": hit.get(\"_id\"),\n \"score\": hit.get(\"_score\"),\n },\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> list[Data]:\n \"\"\"Search documents and return results as a list of Data objects.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in a format that downstream Parser\n components and tool wrappers can consume directly without DataFrame string truncation.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n List containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return []\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return raw_list\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"},"docs_metadata": {"_input_type": "TableInput",diff --git a/src/utils/document_processing.py b/src/utils/document_processing.pyindex a0a9c3996..7d063c1f3 100644--- a/src/utils/document_processing.py+++ b/src/utils/document_processing.py@@ -1,4 +1,5 @@import os+import unicodedatafrom collections import defaultdictfrom utils.logging_config import get_logger@@ -7,6 +8,21 @@ logger = get_logger(__name__)IMAGE_PLACEHOLDER = "<!-- image -->"+# Some PDF fonts map common kanji (日, 月, 木, 金, 土, ...) to the look-alike+# Kangxi Radical code points in their /ToUnicode CMap, and Docling copies them+# through unchanged. Map just those blocks back to CJK Unified Ideographs;+# full NFKC would also rewrite full-width digits, ㈱, ①, half-width katakana.+_CJK_RADICAL_TABLE = {+ cp: unicodedata.normalize("NFKC", chr(cp))+ for cp in range(0x2E80, 0x2FE0) # CJK Radicals Supplement + Kangxi Radicals+ if unicodedata.normalize("NFKC", chr(cp)) != chr(cp)+}+++def normalize_cjk_radicals(text: str) -> str:+ """Replace Kangxi/CJK radical code points with their unified ideographs."""+ return text.translate(_CJK_RADICAL_TABLE)+def process_text_file(file_path: str) -> dict:"""@@ -177,6 +193,9 @@ def extract_relevant(doc_dict: dict) -> dict:})+ for chunk in chunks:+ chunk["text"] = normalize_cjk_radicals(chunk["text"])+return {"id": origin.get("binary_hash"),"filename": origin.get("filename"),diff --git a/tests/unit/test_cjk_radical_normalization.py b/tests/unit/test_cjk_radical_normalization.pynew file mode 100644index 000000000..5650b70e8--- /dev/null+++ b/tests/unit/test_cjk_radical_normalization.py@@ -0,0 +1,48 @@+"""Tests for Kangxi Radical -> CJK Unified Ideograph normalization.++Some PDF fonts map common kanji glyphs (日, 月, 木, 金, 土) to the visually+identical Kangxi Radical code points (U+2F00..U+2FDF) in their /ToUnicode+CMap. Docling copies those code points through unchanged, so search and the+LLM see different characters from the ones a user types.+"""++from src.utils.document_processing import extract_relevant, normalize_cjk_radicals++KANGXI_ROW = "11 ⽉ 27 ⽇ ( ⽊ )" # 11 ⽉ 27 ⽇ ( ⽊ )+++def test_kangxi_radicals_become_unified_ideographs():+ assert normalize_cjk_radicals(KANGXI_ROW) == "11 月 27 日 ( 木 )"+ assert normalize_cjk_radicals("⾦⼟") == "金土"+++def test_every_kangxi_radical_is_mapped():+ radicals = "".join(chr(c) for c in range(0x2F00, 0x2FD6))+ assert not any(0x2F00 <= ord(c) <= 0x2FDF for c in normalize_cjk_radicals(radicals))+++def test_other_compatibility_characters_are_left_alone():+ # Full NFKC would rewrite these; the targeted table must not.+ text = "港区虎ノ門2丁目6-1 ㈱ ① カタカナ (保管中)"+ assert normalize_cjk_radicals(text) == text+++def test_extract_relevant_normalizes_text_and_table_chunks():+ doc = {+ "origin": {"filename": "slip.pdf"},+ "texts": [{"text": "配達⽇", "prov": [{"page_no": 1}]}],+ "tables": [+ {+ "prov": [{"page_no": 1}],+ "data": {+ "table_cells": [+ {"start_row_offset_idx": 0, "start_col_offset_idx": 0, "text": KANGXI_ROW}+ ]+ },+ }+ ],+ }++ texts = [c["text"] for c in extract_relevant(doc)["chunks"]]++ assert texts == ["配達日", "11 月 27 日 ( 木 )"]diff --git a/tests/unit/test_opensearch_embedding_spaces.py b/tests/unit/test_opensearch_embedding_spaces.pyindex a77ba82db..2dd184895 100644--- a/tests/unit/test_opensearch_embedding_spaces.py+++ b/tests/unit/test_opensearch_embedding_spaces.py@@ -350,3 +350,45 @@ def test_direct_ingest_persists_provider_qualified_space(assert captured[0]["embedding_model"] == "text-embedding-3-small"assert captured[0]["embedding_provider"] == "azure"assert captured[0]["embedding_space_id"] == "azure:text-embedding-3-small"+++def test_ingest_normalizes_kangxi_radicals_before_embedding_and_indexing(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ module = _load_opensearch_module(monkeypatch)++ class _Doc:+ text_key = "text"+ default_value = ""++ def model_dump_json(self):+ return '{"text": "11 \\u2f49 27 \\u2f47 ( \\u2f4a )", "filename": "slip.pdf"}'++ class _WatsonxEmbedding:+ model = "ibm/granite-embedding-278m-multilingual"++ def __init__(self):+ self.embedded = []++ def embed_documents(self, texts):+ self.embedded.extend(texts)+ return [[0.1, 0.2] for _ in texts]++ embedding = _WatsonxEmbedding()+ component = _component(module, client=object(), embedding=embedding)+ component.embedding_model_name = ""+ component.docs_metadata = []+ component._prepare_ingest_data = lambda: [_Doc()]+ component._openrag_ingest_callback_config = lambda: ("url", "token", "task")+ indexed = []++ def _bulk_ingest(**kwargs):+ indexed.extend(kwargs["texts"])+ return ["id"]++ component._bulk_ingest_embeddings = _bulk_ingest++ component._add_documents_to_vector_store(object())++ assert embedding.embedded == ["11 月 27 日 ( 木 )"]+ assert indexed == ["11 月 27 日 ( 木 )"]