Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: group acls (#1611)

Sebastián Estévez567231c643b1
diff --git a/.env.example b/.env.example
index 49a260fe5..acbc787b5 100644
--- a/.env.example
+++ b/.env.example
@@ -34,6 +34,10 @@ FETCH_OPENRAG_DOCS_AT_STARTUP=false
# Default: 3600 seconds (60 minutes) total timeout
# INGESTION_TIMEOUT=3600
+# OPTIONAL: OpenSearch JWT TTL for per-user document security tokens (in seconds)
+# Default: INGESTION_TIMEOUT + 300 seconds
+# OPENRAG_OPENSEARCH_JWT_TTL=3900
+
# OPTIONAL: Maximum number of files to upload / ingest (in batch) per task when adding knowledge via folder
# Default: 25
# UPLOAD_BATCH_SIZE=25
diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
index 562582e38..360e1aa1d 100644
--- a/.github/workflows/test-integration.yml
+++ b/.github/workflows/test-integration.yml
@@ -9,6 +9,8 @@ on:
- 'uv.lock'
- 'sdks/**'
- 'flows/**'
+ - 'Makefile'
+ - 'scripts/**'
- '.github/workflows/test-integration.yml'
workflow_dispatch:
inputs:
@@ -18,21 +20,87 @@ on:
type: boolean
default: true
+env:
+ # Prefer repository/environment variable first, then secret, then a sane fallback
+ OPENSEARCH_PASSWORD: ${{ vars.OPENSEARCH_PASSWORD || secrets.OPENSEARCH_PASSWORD || 'OpenRag#2025!' }}
+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+ LANGFLOW_AUTO_LOGIN: "True"
+ LANGFLOW_NEW_USER_IS_ACTIVE: "True"
+ LANGFLOW_ENABLE_SUPERUSER_CLI: "True"
+ LANGFLOW_CHAT_FLOW_ID: ${{ vars.LANGFLOW_CHAT_FLOW_ID || '1098eea1-6649-4e1d-aed1-b77249fb8dd0' }}
+ LANGFLOW_INGEST_FLOW_ID: ${{ vars.LANGFLOW_INGEST_FLOW_ID || '5488df7c-b93f-4f87-a446-b67028bc0813' }}
+ NUDGES_FLOW_ID: ${{ vars.NUDGES_FLOW_ID || 'ebc01d31-1976-46ce-a385-b0240327226c' }}
+ CI_IMAGE_TAG: ci-${{ github.sha }}
+
jobs:
- tests:
+ build-images:
runs-on:
labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"]
- env:
- # Prefer repository/environment variable first, then secret, then a sane fallback
- OPENSEARCH_PASSWORD: ${{ vars.OPENSEARCH_PASSWORD || secrets.OPENSEARCH_PASSWORD || 'OpenRag#2025!' }}
- OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
- LANGFLOW_AUTO_LOGIN: "True"
- LANGFLOW_NEW_USER_IS_ACTIVE: "True"
- LANGFLOW_ENABLE_SUPERUSER_CLI: "True"
- LANGFLOW_CHAT_FLOW_ID: ${{ vars.LANGFLOW_CHAT_FLOW_ID || '1098eea1-6649-4e1d-aed1-b77249fb8dd0' }}
- LANGFLOW_INGEST_FLOW_ID: ${{ vars.LANGFLOW_INGEST_FLOW_ID || '5488df7c-b93f-4f87-a446-b67028bc0813' }}
- NUDGES_FLOW_ID: ${{ vars.NUDGES_FLOW_ID || 'ebc01d31-1976-46ce-a385-b0240327226c' }}
+ steps:
+ - run: df -h
+
+ - name: Cleanup Docker cache
+ run: |
+ docker system prune -af || true
+ docker builder prune -af || true
+ docker compose -f docker-compose.yml down -v --remove-orphans || true
+
+ - name: Cleanup root-owned files (OpenSearch data, config, Langflow data, keys, data, flows, documents)
+ run: |
+ for i in 1 2 3; do
+ docker run --rm -v $(pwd):/work alpine sh -c "rm -rf /work/opensearch-data /work/config /work/langflow-data /work/keys /work/data /work/flows /work/openrag-documents" && break
+ echo "Attempt $i failed, retrying in 5s..."
+ sleep 5
+ done || true
+
+ - run: df -h
+
+ - name: Checkout
+ uses: actions/checkout@v4
+ - name: Verify workspace
+ run: |
+ echo "Current directory: $(pwd)"
+ echo "Workspace: ${GITHUB_WORKSPACE}"
+ ls -la
+
+ - name: Build OpenRAG images
+ run: |
+ USE_LOCAL="${{ inputs.use_local_images }}"
+ if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ]; then
+ echo "Building PR images locally..."
+ OPENRAG_VERSION="${CI_IMAGE_TAG}" make ci-build-images
+ else
+ echo "Preparing DockerHub images..."
+ docker pull langflowai/openrag-backend:latest
+ docker pull langflowai/openrag-frontend:latest
+ docker pull langflowai/openrag-langflow:latest
+ docker tag langflowai/openrag-backend:latest "langflowai/openrag-backend:${CI_IMAGE_TAG}"
+ docker tag langflowai/openrag-frontend:latest "langflowai/openrag-frontend:${CI_IMAGE_TAG}"
+ docker tag langflowai/openrag-langflow:latest "langflowai/openrag-langflow:${CI_IMAGE_TAG}"
+ docker build --no-cache -t "langflowai/openrag-opensearch:${CI_IMAGE_TAG}" -f Dockerfile .
+ fi
+
+ - name: Save OpenRAG images
+ run: OPENRAG_VERSION="${CI_IMAGE_TAG}" make ci-save-images
+
+ - name: Upload OpenRAG image artifact
+ uses: actions/upload-artifact@v4
+ with:
+ name: openrag-ci-images-${{ github.sha }}
+ path: .ci-artifacts/openrag-ci-images.tar
+ retention-days: 1
+ compression-level: 0
+
+ test-suite:
+ needs: build-images
+ runs-on:
+ labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"]
+ strategy:
+ fail-fast: false
+ matrix:
+ suite: [core, sdk-python, sdk-typescript]
+ name: tests (${{ matrix.suite }})
steps:
- run: df -h
@@ -54,7 +122,7 @@ jobs:
- name: Checkout
uses: actions/checkout@v4
-
+
- name: Verify workspace
run: |
echo "Current directory: $(pwd)"
@@ -67,6 +135,7 @@ jobs:
version: latest
- name: Set up Node.js
+ if: matrix.suite == 'sdk-typescript'
uses: actions/setup-node@v4
with:
node-version: '20'
@@ -74,10 +143,16 @@ jobs:
- name: Python version
run: uv python install 3.13
- - name: Install dependencies
- run: uv sync
+ - name: Download OpenRAG image artifact
+ uses: actions/download-artifact@v4
+ with:
+ name: openrag-ci-images-${{ github.sha }}
+ path: .ci-artifacts
- - name: Run integration tests
+ - name: Load OpenRAG images
+ run: docker load -i .ci-artifacts/openrag-ci-images.tar
+
+ - name: Run ${{ matrix.suite }} integration suite
env:
OPENSEARCH_HOST: localhost
OPENSEARCH_PORT: 9200
@@ -90,15 +165,24 @@ jobs:
# Disable startup ingest noise unless a test enables it
DISABLE_STARTUP_INGEST: "true"
run: |
- # For PRs, always build locally since we're testing new code
- # For workflow_dispatch, use the input (defaults to true)
- USE_LOCAL="${{ inputs.use_local_images }}"
- if [ "${{ github.event_name }}" == "pull_request" ] || [ "$USE_LOCAL" != "false" ]; then
- echo "Running tests with locally built images..."
- make test-ci-local
- else
- echo "Running tests with DockerHub images..."
- make test-ci
- fi
+ OPENRAG_VERSION="${CI_IMAGE_TAG}" TEST_SUITE="${{ matrix.suite }}" make test-ci-suite
echo "Keys directory after tests:"
ls -la keys/ || echo "No keys directory"
+
+ tests:
+ needs: [build-images, test-suite]
+ if: always()
+ runs-on:
+ labels: ["self-hosted", "linux", "ARM64", "langflow-ai-arm64-40gb-ephemeral-sudo"]
+ steps:
+ - name: Check split integration result
+ run: |
+ if [ "${{ needs.build-images.result }}" != "success" ]; then
+ echo "build-images failed"
+ exit 1
+ fi
+ if [ "${{ needs.test-suite.result }}" != "success" ]; then
+ echo "one or more integration suites failed"
+ exit 1
+ fi
+ echo "all integration suites passed"
diff --git a/AGENTS.md b/AGENTS.md
index 230456946..79696881f 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -44,7 +44,7 @@ Read the `SKILL.md` files directly. The frontmatter `description` tells you when
**RBAC is opt-in.** `OPENRAG_RBAC_ENFORCE` defaults to `false`, which makes OpenRAG behave like the pre-RBAC release: every authenticated user has full access; API-key role overrides are also bypassed. To turn the permissions system on (admin/developer/user/viewer roles, `require_permission` gates, audit denials), set `OPENRAG_RBAC_ENFORCE=true`. Available in all `OPENRAG_RUN_MODE` values — operators own the trade-off. The startup event logs the enforcement state on every boot.
-**Dev-local with backend on host.** When you run `make dev-local-cpu` (or `dev-local`) and then `make backend` on the host, OpenSearch needs to resolve `openrag-backend` to the host machine so it can fetch JWKS for OIDC validation. The base `docker-compose.yml` does NOT add this alias — it would break CI, where the backend is a docker-compose service. To enable the host-backend mode, layer the override file:
+**Dev-local with backend on host.** When you run `make dev-local-cpu` (or `dev-local`) and then `make backend` on the host, OpenSearch needs to resolve `openrag-backend` to the host machine so it can fetch JWKS for OIDC validation. Langflow also needs that name for backend ingest callbacks. The base `docker-compose.yml` does NOT add this alias — it would break CI, where the backend is a docker-compose service. To enable the host-backend mode, layer the override file:
```bash
docker compose -f docker-compose.yml -f docker-compose.host-backend.yml up -d opensearch dashboards langflow
@@ -52,4 +52,4 @@ make backend # in another terminal
make frontend # in another terminal
```
-The override only adds `extra_hosts: openrag-backend:host-gateway` to the OpenSearch service. Without it, OIDC works because docker DNS routes `openrag-backend` to the in-compose backend container.
+The override only adds `extra_hosts: openrag-backend:host-gateway` to the OpenSearch and Langflow services. Without it, OIDC and ingest callbacks work because docker DNS routes `openrag-backend` to the in-compose backend container.
diff --git a/Makefile b/Makefile
index c50b72014..d8ce272b4 100644
--- a/Makefile
+++ b/Makefile
@@ -97,7 +97,8 @@ endef
.PHONY: help check_tools help_docker help_dev help_test help_local help_utils help_operator \
dev dev-cpu dev-local dev-local-cpu dev-local-build-lf dev-local-build-lf-cpu stop clean build logs \
shell-backend shell-frontend install \
- test test-unit test-integration test-ci test-ci-local test-sdk test-os-jwt lint \
+ test test-unit test-integration test-ci test-ci-local test-ci-suite test-sdk test-os-jwt lint \
+ ci-build-images ci-save-images \
backend frontend docling docling-stop install-be install-fe build-be build-fe build-os build-lf logs-be logs-fe logs-lf logs-os \
shell-be shell-lf shell-os restart status health db-reset clear-os-data flow-upload setup factory-reset \
dev-branch build-langflow-dev stop-dev clean-dev logs-dev logs-lf-dev shell-lf-dev restart-dev status-dev \
@@ -782,6 +783,30 @@ test-integration: ## Run integration tests (requires infrastructure)
@echo "$(YELLOW)Make sure to run 'make dev-local' first!$(NC)"
uv run pytest tests/integration/core/ -v
+ci-build-images: ## Build all OpenRAG images for CI artifact sharing
+ @set -e; \
+ IMAGE_TAG=$${OPENRAG_VERSION:-latest}; \
+ echo "$(YELLOW)Building all OpenRAG images with tag '$$IMAGE_TAG'...$(NC)"; \
+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-opensearch:$$IMAGE_TAG -f Dockerfile .; \
+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-backend:$$IMAGE_TAG -f Dockerfile.backend .; \
+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-frontend:$$IMAGE_TAG -f Dockerfile.frontend .; \
+ $(CONTAINER_RUNTIME) build -t langflowai/openrag-langflow:$$IMAGE_TAG -f Dockerfile.langflow .
+
+ci-save-images: ## Save CI-built OpenRAG images to .ci-artifacts/openrag-ci-images.tar
+ @set -e; \
+ IMAGE_TAG=$${OPENRAG_VERSION:-latest}; \
+ mkdir -p .ci-artifacts; \
+ echo "$(YELLOW)Saving OpenRAG images with tag '$$IMAGE_TAG'...$(NC)"; \
+ $(CONTAINER_RUNTIME) save -o .ci-artifacts/openrag-ci-images.tar \
+ langflowai/openrag-opensearch:$$IMAGE_TAG \
+ langflowai/openrag-backend:$$IMAGE_TAG \
+ langflowai/openrag-frontend:$$IMAGE_TAG \
+ langflowai/openrag-langflow:$$IMAGE_TAG; \
+ ls -lh .ci-artifacts/openrag-ci-images.tar
+
+test-ci-suite: ensure-langflow-data ensure-backend-volumes ## Run one CI integration suite: TEST_SUITE=core|sdk-python|sdk-typescript
+ @scripts/ci/run_integration_suite.sh "$${TEST_SUITE:-core}"
+
test-ci: ensure-langflow-data ensure-backend-volumes ## Start infra, run integration + SDK tests, tear down (uses DockerHub images)
@set -e; \
echo "$(YELLOW)Installing test dependencies...$(NC)"; \
@@ -871,6 +896,7 @@ test-ci: ensure-langflow-data ensure-backend-volumes ## Start infra, run integra
GOOGLE_OAUTH_CLIENT_ID="" \
GOOGLE_OAUTH_CLIENT_SECRET="" \
OPENSEARCH_HOST=localhost OPENSEARCH_PORT=9200 \
+ LANGFLOW_OPENSEARCH_HOST=opensearch LANGFLOW_OPENSEARCH_PORT=9200 \
OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD=$${OPENSEARCH_PASSWORD} \
DISABLE_STARTUP_INGEST=$${DISABLE_STARTUP_INGEST:-true} \
uv run pytest tests/integration/core -vv -s -o log_cli=true --log-cli-level=DEBUG; \
@@ -995,6 +1021,7 @@ test-ci-local: ensure-langflow-data ensure-backend-volumes ## Same as test-ci bu
GOOGLE_OAUTH_CLIENT_ID="" \
GOOGLE_OAUTH_CLIENT_SECRET="" \
OPENSEARCH_HOST=localhost OPENSEARCH_PORT=9200 \
+ LANGFLOW_OPENSEARCH_HOST=opensearch LANGFLOW_OPENSEARCH_PORT=9200 \
OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD=$${OPENSEARCH_PASSWORD} \
DISABLE_STARTUP_INGEST=$${DISABLE_STARTUP_INGEST:-true} \
uv run pytest tests/integration/core -vv -s -o log_cli=true --log-cli-level=DEBUG; \
diff --git a/cloud_securityconfig/roles.yml b/cloud_securityconfig/roles.yml
index 7f38a6a7c..460e39169 100644
--- a/cloud_securityconfig/roles.yml
+++ b/cloud_securityconfig/roles.yml
@@ -3,55 +3,44 @@ _meta:
config_version: 2
openrag_user_role:
- description: "DLS: user can read/write docs they own or are allowed on"
+ description: "DLS: user can read docs they own or are allowed on"
cluster_permissions:
- - "indices:data/write/bulk"
- - "indices:data/write/index"
- "indices:data/read/scroll"
- "indices:data/read/scroll/clear"
- "cluster:monitor/*"
- - "cluster:admin/opensearch/notifications/configs/create"
- - "cluster:admin/opensearch/notifications/configs/list"
- - "cluster:admin/opensearch/notifications/configs/get"
- - "cluster:admin/opensearch/notifications/configs/update"
- - "cluster:admin/opensearch/notifications/configs/delete"
- - "cluster:admin/opensearch/alerting/*"
index_permissions:
- - index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*", "orag-*",orag*]
+ - index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*", "orag-*", "orag*"]
allowed_actions:
- - crud
- - create_index
+ - read
- indices:admin/mappings/get
- - indices:admin/mappings/put
- indices:admin/exists
- indices:admin/get
- - indices:admin/refresh
- - indices:data/write/delete/byquery
- - indices:data/write/update/byquery
dls: >
{"bool":{"should":[
{"term":{"owner":"${user.name}"}},
+ {"term":{"owner":"${attr.jwt.email}"}},
{"term":{"allowed_users":"${user.name}"}},
+ {"term":{"allowed_users":"${attr.jwt.email}"}},
+ {"terms":{"allowed_principals":{"index":"openrag_dls_principals","id":"${user.name}","path":"principals"}}},
{"bool":{"must_not":{"exists":{"field":"owner"}}}}
],"minimum_should_match":1}}
+ - index_patterns: ["openrag_dls_principals", "openrag_dls_principals*"]
+ allowed_actions:
+ - read
+ - indices:admin/exists
+ - indices:data/read/get
+ - indices:data/read/search
+ dls: >
+ {"term":{"user_name":"${user.name}"}}
- index_patterns: ["api_keys", "api_keys*"]
allowed_actions:
- - crud
- - create_index
+ - read
- indices:admin/mappings/get
- - indices:admin/mappings/put
- indices:admin/exists
- indices:admin/get
- - indices:admin/refresh
- - indices:data/write/delete/byquery
- - indices:data/write/update/byquery
dls: >
{"bool":{"should":[
{"term":{"user_id":"${user.name}"}},
{"bool":{"must_not":{"exists":{"field":"user_id"}}}}
],"minimum_should_match":1}}
- - index_patterns: [".opendistro-alerting-config"]
- allowed_actions:
- - crud
- - indices:admin/get
diff --git a/docker-compose.host-backend.yml b/docker-compose.host-backend.yml
index d2734a9ea..19806914f 100644
--- a/docker-compose.host-backend.yml
+++ b/docker-compose.host-backend.yml
@@ -28,3 +28,6 @@ services:
opensearch:
extra_hosts:
- "openrag-backend:host-gateway"
+ langflow:
+ extra_hosts:
+ - "openrag-backend:host-gateway"
diff --git a/docker-compose.yml b/docker-compose.yml
index 3db1c5a7b..1c5d06ae0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,6 +56,7 @@ services:
environment:
- OPENSEARCH_HOST=${OPENSEARCH_HOST:-opensearch}
- LANGFLOW_URL=http://langflow:7860
+ - OPENRAG_BACKEND_INTERNAL_URL=${OPENRAG_BACKEND_INTERNAL_URL:-http://openrag-backend:8000}
- LANGFLOW_PUBLIC_URL=${LANGFLOW_PUBLIC_URL}
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
- LANGFLOW_SUPERUSER=${LANGFLOW_SUPERUSER}
@@ -101,7 +102,7 @@ services:
- LANGFLOW_VERSION=${LANGFLOW_VERSION}
- LOG_FORMAT=${LOG_FORMAT}
- SERVICE_NAME=${SERVICE_NAME}
- - SESSION_SECRET=${SESSION_SECRET}
+ - SESSION_SECRET=${SESSION_SECRET:-your-secret-key-change-in-production}
- JWT_SIGNING_KEY=${JWT_SIGNING_KEY}
- OPENRAG_ENCRYPTION_KEY=${OPENRAG_ENCRYPTION_KEY}
- LOG_LEVEL=${LOG_LEVEL:-INFO}
@@ -182,7 +183,12 @@ services:
- SOURCE_URL=None
- ALLOWED_USERS=[]
- ALLOWED_GROUPS=[]
+ - ALLOWED_PRINCIPALS=[]
- OPENRAG-QUERY-FILTER="{}"
+ - OPENRAG_INGEST_URL=OPENRAG_INGEST_URL
+ - OPENRAG_INGEST_TOKEN=OPENRAG_INGEST_TOKEN
+ - OPENRAG_INGEST_RUN_ID=OPENRAG_INGEST_RUN_ID
+ - OPENRAG_INGEST_BATCH_SIZE=100
- OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
- OPENSEARCH_HOST=${LANGFLOW_OPENSEARCH_HOST:-${OPENSEARCH_HOST:-opensearch}}
- OPENSEARCH_PORT=${LANGFLOW_OPENSEARCH_PORT:-${OPENSEARCH_PORT:-9200}}
@@ -194,7 +200,7 @@ services:
- MIMETYPE=None
- FILESIZE=0
- SELECTED_EMBEDDING_MODEL=${SELECTED_EMBEDDING_MODEL:-text-embedding-3-small}
- - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,DOCLING_TASK_ID,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_APIKEY,WATSONX_URL,WATSONX_PROJECT_ID,OLLAMA_BASE_URL,OPENSEARCH_INDEX_NAME
+ - LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT=JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,DOCLING_SERVE_URL,DOCLING_TASK_ID,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,ALLOWED_PRINCIPALS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_APIKEY,WATSONX_URL,WATSONX_PROJECT_ID,OLLAMA_BASE_URL,OPENSEARCH_INDEX_NAME,OPENRAG_INGEST_URL,OPENRAG_INGEST_TOKEN,OPENRAG_INGEST_RUN_ID,OPENRAG_INGEST_BATCH_SIZE
- LANGFLOW_LOG_LEVEL=DEBUG
- LANGFLOW_WORKERS=${LANGFLOW_WORKERS:-1}
- LANGFLOW_AUTO_LOGIN=${LANGFLOW_AUTO_LOGIN}
diff --git a/flows/components/opensearch_multimodal.py b/flows/components/opensearch_multimodal.py
index b77eafd38..ed609526c 100644
--- a/flows/components/opensearch_multimodal.py
+++ b/flows/components/opensearch_multimodal.py
@@ -6,6 +6,7 @@ import uuid
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Any
+import httpx
from lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
from lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection
from lfx.io import (
@@ -158,7 +159,16 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
"docs_metadata",
"request_timeout",
"max_retries",
+ "openrag_ingest_url",
+ "openrag_ingest_token",
+ "openrag_ingest_run_id",
+ "openrag_ingest_batch_size",
]
+ _openrag_ingest_global_placeholders = {
+ "openrag_ingest_url": "OPENRAG_INGEST_URL",
+ "openrag_ingest_token": "OPENRAG_INGEST_TOKEN",
+ "openrag_ingest_run_id": "OPENRAG_INGEST_RUN_ID",
+ }
inputs = [
TableInput(
@@ -185,6 +195,23 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
value=[],
input_types=["Data", "JSON"],
),
+ StrInput(
+ name="openrag_ingest_token",
+ display_name="OpenRAG Ingest Token",
+ value="OPENRAG_INGEST_TOKEN",
+ load_from_db=True,
+ input_types=["Text", "Message"],
+ advanced=True,
+ info="Short-lived token used only for OpenRAG ingest callbacks.",
+ ),
+ StrInput(
+ name="openrag_ingest_run_id",
+ display_name="OpenRAG Ingest Run ID",
+ value="OPENRAG_INGEST_RUN_ID",
+ load_from_db=True,
+ input_types=["Text", "Message"],
+ advanced=True,
+ ),
StrInput(
name="opensearch_url",
display_name="OpenSearch URL",
@@ -257,7 +284,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
advanced=True,
),
*LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.
- HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"], is_list=True),
+ HandleInput(
+ name="embedding", display_name="Embedding", input_types=["Embeddings"], is_list=True
+ ),
StrInput(
name="embedding_model_name",
display_name="Embedding Model Name",
@@ -309,11 +338,13 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
DropdownInput(
name="auth_mode",
display_name="Authentication Mode",
- value="jwt",
- options=["basic", "jwt"],
+ value="openrag",
+ options=["basic", "jwt", "openrag"],
info=(
"Authentication method: 'basic' for username/password authentication, "
- "or 'jwt' for JSON Web Token (Bearer) authentication."
+ "'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to "
+ "delegate writes to the OpenRAG backend ingest callback (no direct "
+ "OpenSearch credentials required — only OPENRAG_* fields)."
),
real_time_refresh=True,
advanced=False,
@@ -340,6 +371,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
"Valid JSON Web Token for authentication. "
"Will be sent in the Authorization header (with optional 'Bearer ' prefix)."
),
+ required=False
),
StrInput(
name="jwt_header",
@@ -391,6 +423,38 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
advanced=True,
info="Number of retries for failed connections before raising an error.",
),
+ StrInput(
+ name="openrag_ingest_url",
+ display_name="OpenRAG Ingest URL",
+ value="OPENRAG_INGEST_URL",
+ load_from_db=True,
+ input_types=["Text", "Message"],
+ advanced=True,
+ info="Internal OpenRAG callback URL for backend-owned document indexing.",
+ ),
+ StrInput(
+ name="openrag_ingest_token",
+ display_name="OpenRAG Ingest Token",
+ value="OPENRAG_INGEST_TOKEN",
+ load_from_db=True,
+ input_types=["Text", "Message"],
+ advanced=True,
+ info="Short-lived token used only for OpenRAG ingest callbacks.",
+ ),
+ StrInput(
+ name="openrag_ingest_run_id",
+ display_name="OpenRAG Ingest Run ID",
+ value="OPENRAG_INGEST_RUN_ID",
+ load_from_db=True,
+ input_types=["Text", "Message"],
+ advanced=True,
+ ),
+ IntInput(
+ name="openrag_ingest_batch_size",
+ display_name="OpenRAG Ingest Batch Size",
+ value=100,
+ advanced=True,
+ ),
]
outputs = [
Output(
@@ -472,8 +536,10 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
# Apply score_threshold / scoreThreshold as min_score if not already set
if "min_score" not in query_body:
+
score_threshold = self._resolve_score_threshold(filter_obj)
if score_threshold is not None:
+
query_body["min_score"] = score_threshold
client = self.build_client()
@@ -490,7 +556,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
def is_vector(val):
# Accepts if it's a list of numbers (float or int) and has reasonable vector length
return (
- isinstance(val, list) and len(val) > min_vector_length and all(isinstance(x, (float, int)) for x in val)
+ isinstance(val, list)
+ and len(val) > min_vector_length
+ and all(isinstance(x, (float, int)) for x in val)
)
if "hits" in resp and "hits" in resp["hits"]:
@@ -712,7 +780,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
)
# Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist
return
- logger.warning(f"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}")
+ logger.warning(
+ f"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}"
+ )
raise
# Verify the field was added correctly
@@ -748,7 +818,158 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
Returns:
True if AOSS is enabled, False otherwise
"""
- return http_auth is not None and hasattr(http_auth, "service") and http_auth.service == "aoss"
+ return (
+ http_auth is not None and hasattr(http_auth, "service") and http_auth.service == "aoss"
+ )
+
+ @staticmethod
+ def _openrag_input_to_str(value: Any) -> str:
+ if value is None:
+ return ""
+ if hasattr(value, "get_secret_value"):
+ value = value.get_secret_value()
+ if hasattr(value, "text"):
+ value = value.text
+ return str(value or "").strip()
+
+ def _openrag_callback_value(self, attr_name: str) -> str:
+ value = self._openrag_input_to_str(getattr(self, attr_name, ""))
+ if value == self._openrag_ingest_global_placeholders.get(attr_name):
+ return ""
+ return value
+
+ def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:
+ url = self._openrag_callback_value("openrag_ingest_url")
+ token = self._openrag_callback_value("openrag_ingest_token")
+ ingest_run_id = self._openrag_callback_value("openrag_ingest_run_id")
+
+ masked_token = (
+ f"{token[:4]}...{token[-4:]}" if len(token) >= 8 else ("<set>" if token else "")
+ )
+ debug_payload = {
+ "openrag_ingest_url": url,
+ "openrag_ingest_url_len": len(url),
+ "openrag_ingest_token_masked": masked_token,
+ "openrag_ingest_token_len": len(token),
+ "openrag_ingest_run_id": ingest_run_id,
+ "raw_url_type": type(self.openrag_ingest_url).__name__,
+ "raw_token_type": type(self.openrag_ingest_token).__name__,
+ "raw_run_id_type": type(self.openrag_ingest_run_id).__name__,
+ }
+ logger.warning(f"[OpenRAG callback config] {debug_payload}")
+ try:
+ self.log(f"[OpenRAG callback config] {debug_payload}")
+ except Exception:
+ pass
+
+ if not url and not token and not ingest_run_id:
+ return None
+ if not url or not token or not ingest_run_id:
+ msg = "OpenRAG ingest callback requires url, token, and ingest_run_id."
+ raise ValueError(msg)
+ return url, token, ingest_run_id
+
+ def _post_openrag_ingest_batches(
+ self,
+ *,
+ requests: list[dict],
+ vector_field: str,
+ text_field: str,
+ ) -> None:
+ callback_config = self._openrag_ingest_callback_config()
+ if callback_config is None:
+ return
+
+ url, token, ingest_run_id = callback_config
+ batch_size = max(self._parse_int_param("openrag_ingest_batch_size", 100), 1)
+ timeout = self._parse_int_param("request_timeout", REQUEST_TIMEOUT)
+ headers = {"Authorization": f"Bearer {token}"}
+
+ masked_token = (
+ f"{token[:4]}...{token[-4:]}" if len(token) >= 8 else ("<set>" if token else "")
+ )
+ request_summary = {
+ "url": url,
+ "ingest_run_id": ingest_run_id,
+ "token_masked": masked_token,
+ "total_chunks": len(requests),
+ "batch_size": batch_size,
+ "timeout_s": timeout,
+ }
+ logger.warning(f"[OpenRAG ingest POST] {request_summary}")
+ try:
+ self.log(f"[OpenRAG ingest POST] {request_summary}")
+ except Exception:
+ pass
+
+ with httpx.Client(timeout=timeout) as client:
+ total_batches = (len(requests) + batch_size - 1) // batch_size
+ for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):
+ batch = requests[start : start + batch_size]
+ final = batch_number == total_batches
+ payload = {
+ "ingest_run_id": ingest_run_id,
+ "batch_id": batch_number,
+ "final": final,
+ "chunks": [
+ self._openrag_chunk_payload(
+ request,
+ vector_field=vector_field,
+ text_field=text_field,
+ )
+ for request in batch
+ ],
+ }
+ logger.warning(
+ f"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} "
+ f"url={url} chunks={len(payload['chunks'])} final={final}"
+ )
+ response = client.post(url, json=payload, headers=headers)
+ response_summary = {
+ "batch": batch_number,
+ "url": url,
+ "status": response.status_code,
+ "final_url": str(response.request.url),
+ "response_headers": dict(response.headers),
+ "body_preview": response.text[:500],
+ }
+ logger.warning(f"[OpenRAG ingest POST resp] {response_summary}")
+ try:
+ self.log(f"[OpenRAG ingest POST resp] {response_summary}")
+ except Exception:
+ pass
+ if response.status_code >= 400:
+ msg = (
+ "OpenRAG ingest callback failed "
+ f"(batch={batch_number}, status={response.status_code}, "
+ f"url={url}): {response.text[:1000]}"
+ )
+ raise RuntimeError(msg)
+
+ self.log(f"Posted {len(requests)} chunks to OpenRAG backend ingest callback.")
+
+ @staticmethod
+ def _openrag_chunk_payload(
+ request: dict,
+ *,
+ vector_field: str,
+ text_field: str,
+ ) -> dict:
+ metadata = {
+ key: value
+ for key, value in request.items()
+ if key not in {"_op_type", "_index", "_id", "id", vector_field, text_field}
+ }
+ page = metadata.get("page")
+ if isinstance(page, str) and page.isdigit():
+ page = int(page)
+ return {
+ "id": request.get("_id") or request.get("id"),
+ "text": request.get(text_field, ""),
+ "vector": request[vector_field],
+ "page": page if isinstance(page, int) else None,
+ "metadata": metadata,
+ }
def _bulk_ingest_embeddings(
self,
@@ -803,7 +1024,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
metadata = {**metadata, "embedding_dimensions": vector_dimensions}
# Normalize ACL fields that may arrive as JSON strings from flows
- for key in ("allowed_users", "allowed_groups"):
+ for key in ("allowed_users", "allowed_groups", "allowed_principals"):
value = metadata.get(key)
if isinstance(value, str):
try:
@@ -814,7 +1035,12 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
# Leave value as-is if it isn't valid JSON
pass
- _id = ids[i] if ids else str(uuid.uuid4())
+ metadata_document_id = str(metadata.get("document_id") or "").strip()
+ if metadata_document_id and metadata_document_id.lower() != "none":
+ generated_id = f"{metadata_document_id}_{i}"
+ else:
+ generated_id = str(uuid.uuid4())
+ _id = ids[i] if ids else generated_id
request = {
"_op_type": "index",
"_index": index_name,
@@ -831,9 +1057,47 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
return_ids.append(_id)
if metadatas:
self.log(f"Sample metadata: {metadatas[0] if metadatas else {}}")
- helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)
+ if self._openrag_ingest_callback_config() is not None:
+ self._post_openrag_ingest_batches(
+ requests=requests,
+ vector_field=vector_field,
+ text_field=text_field,
+ )
+ return return_ids
+ try:
+ helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)
+ except Exception as bulk_error:
+ if "Unsupported request type for filter level DLS" not in str(bulk_error):
+ raise
+ logger.warning(
+ "[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; "
+ "falling back to per-document index requests."
+ )
+ self._index_embeddings_individually(client, requests)
return return_ids
+ def _index_embeddings_individually(
+ self,
+ client: OpenSearch,
+ requests: list[dict],
+ ) -> None:
+ """Index documents one at a time when OpenSearch DLS rejects bulk writes."""
+ for request in requests:
+ document_id = request.get("_id") or request.get("id")
+ body = {
+ key: value
+ for key, value in request.items()
+ if key not in {"_op_type", "_index", "_id", "id"}
+ }
+ client.index(index=request["_index"], id=document_id, body=body)
+
+ def _log_index_admin_skip(self, operation: str, error: Exception) -> None:
+ """Log index-admin operations that may be blocked under filter-level DLS."""
+ logger.warning(
+ f"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. "
+ "Assuming the backend pre-created the required index/mapping and continuing."
+ )
+
# ---------- param helpers ----------
def _parse_int_param(self, attr_name: str, default: int) -> int:
"""Parse a string attribute to int, returning *default* on failure."""
@@ -843,7 +1107,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
try:
value = int(str(raw).strip())
except ValueError:
- logger.warning(f"Invalid integer value '{raw}' for {attr_name}, using default {default}")
+ logger.warning(
+ f"Invalid integer value '{raw}' for {attr_name}, using default {default}"
+ )
return default
if value < 0:
@@ -874,6 +1140,26 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
header_name = (self.jwt_header or "Authorization").strip()
header_value = f"Bearer {token}" if self.bearer_prefix else token
return {"headers": {header_name: header_value}}
+ if mode == "openrag":
+ # Writes are delegated to the OpenRAG backend ingest callback,
+ # so no direct OpenSearch credentials are needed. Only the
+ # OPENRAG_* fields are required for ingestion to function.
+ missing = [
+ name
+ for name, value in (
+ ("openrag_ingest_url", self.openrag_ingest_url),
+ ("openrag_ingest_token", self.openrag_ingest_token),
+ ("openrag_ingest_run_id", self.openrag_ingest_run_id),
+ )
+ if not (value or "").strip()
+ ]
+ if missing:
+ msg = (
+ "Auth Mode is 'openrag' but required OPENRAG_* fields are "
+ f"missing: {', '.join(missing)}."
+ )
+ raise ValueError(msg)
+ return {}
user = (self.username or "").strip()
pwd = (self.password or "").strip()
if not user or not pwd:
@@ -909,7 +1195,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
# Check if we're in ingestion-only mode (no search query)
has_search_query = bool((self.search_query or "").strip())
if not has_search_query:
- logger.debug("[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped")
+ logger.debug(
+ "[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped"
+ )
logger.debug("[OpenSearchMultimodel] Starting ingestion mode...")
logger.debug(f"[OpenSearchMultimodel] Embedding: {self.embedding}")
@@ -959,19 +1247,31 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
# NOW check if we have any valid embeddings left after filtering
if not embeddings_list:
- logger.warning("All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.")
- self.log("Embedding returned None (fail-safe mode enabled). Skipping document ingestion.")
+ logger.warning(
+ "All embeddings returned None (fail-safe mode enabled). Skipping document ingestion."
+ )
+ self.log(
+ "Embedding returned None (fail-safe mode enabled). Skipping document ingestion."
+ )
return
- logger.debug(f"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}")
- self.log(f"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}")
+ logger.debug(
+ f"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}"
+ )
+ self.log(
+ f"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}"
+ )
# Select the embedding to use for ingestion
selected_embedding = None
embedding_model = None
# If embedding_model_name is specified, find matching embedding
- if hasattr(self, "embedding_model_name") and self.embedding_model_name and self.embedding_model_name.strip():
+ if (
+ hasattr(self, "embedding_model_name")
+ and self.embedding_model_name
+ and self.embedding_model_name.strip()
+ ):
target_model_name = self.embedding_model_name.strip()
self.log(f"Looking for embedding model: {target_model_name}")
@@ -1017,12 +1317,16 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
# Use the dedicated embedding instance from the dict
selected_embedding = available_models_attr[target_model_name]
embedding_model = target_model_name
- self.log(f"Found dedicated embedding instance for '{embedding_model}' in available_models dict")
+ self.log(
+ f"Found dedicated embedding instance for '{embedding_model}' in available_models dict"
+ )
else:
# Traditional identifier match
selected_embedding = emb_obj
embedding_model = self._get_embedding_model_name(emb_obj)
- self.log(f"Found matching embedding model: {embedding_model} (matched on: {target_model_name})")
+ self.log(
+ f"Found matching embedding model: {embedding_model} (matched on: {target_model_name})"
+ )
break
if not selected_embedding:
@@ -1093,7 +1397,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
# No model switching needed - each model in available_models has its own dedicated instance
# The selected_embedding is already configured correctly for the target model
- logger.info(f"Using embedding instance for '{embedding_model}' - pre-configured and ready to use")
+ logger.info(
+ f"Using embedding instance for '{embedding_model}' - pre-configured and ready to use"
+ )
# Extract texts and metadata from documents
texts = []
@@ -1140,6 +1446,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
logger.debug(f"Is IBM/watsonx embedding: {is_ibm}")
if is_ibm:
+
# Hand the full batch to the SDK and let it batch/throttle/retry.
# Retry attempts and base backoff are tunable via the SDK's own
# WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.
@@ -1155,6 +1462,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
f"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}",
)
raise
+
else:
# Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's
# built-in rate-limit handling, so embed per chunk in parallel with
@@ -1223,7 +1531,10 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
logger.debug(f"Using parallel processing with {max_workers} workers")
with ThreadPoolExecutor(max_workers=max_workers) as executor:
- futures = {executor.submit(embed_chunk_with_retry, chunk, idx): idx for idx, chunk in enumerate(texts)}
+ futures = {
+ executor.submit(embed_chunk_with_retry, chunk, idx): idx
+ for idx, chunk in enumerate(texts)
+ }
for future in as_completed(futures):
idx = futures[future]
vectors[idx] = future.result()
@@ -1234,68 +1545,84 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
# Get vector dimension for mapping
dim = len(vectors[0]) if vectors else 768 # default fallback
+ use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None
- # Check for AOSS
- auth_kwargs = self._build_auth_kwargs()
- is_aoss = self._is_aoss_enabled(auth_kwargs.get("http_auth"))
+ is_aoss = False
+ mapping: dict | None = None
- # Validate engine with AOSS
engine = getattr(self, "engine", "jvector")
- self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)
-
- # Create mapping with proper KNN settings
- space_type = getattr(self, "space_type", "l2")
- ef_construction = getattr(self, "ef_construction", 512)
- m = getattr(self, "m", 16)
-
- mapping = self._default_text_mapping(
- dim=dim,
- engine=engine,
- space_type=space_type,
- ef_construction=ef_construction,
- m=m,
- vector_field=dynamic_field_name, # Use dynamic field name
- )
- # Ensure index exists with baseline mapping (index.knn: true is required for vector search)
- try:
- if not client.indices.exists(index=self.index_name):
- self.log(f"Creating index '{self.index_name}' with base mapping")
- client.indices.create(index=self.index_name, body=mapping)
- except RequestError as creation_error:
- if creation_error.error == "resource_already_exists_exception":
- pass # Index was created concurrently
- else:
- error_msg = str(creation_error).lower()
- if "invalid engine" in error_msg or "illegal_argument" in error_msg:
- if "jvector" in error_msg:
- msg = (
- "The 'jvector' engine is not available in your OpenSearch installation. "
- "Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+."
- )
- raise ValueError(msg) from creation_error
- if "index.knn" in error_msg:
- msg = (
- "The index has index.knn: false. Delete the existing index and let the "
- "component recreate it, or create a new index with a different name."
- )
- raise ValueError(msg) from creation_error
- logger.warning(f"Failed to create index '{self.index_name}': {creation_error}")
- raise
+ if use_openrag_ingest_callback:
+ self.log("Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.")
+ else:
+ # Check for AOSS
+ auth_kwargs = self._build_auth_kwargs()
+ is_aoss = self._is_aoss_enabled(auth_kwargs.get("http_auth"))
+
+ # Validate engine with AOSS
+ self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)
+
+ # Create mapping with proper KNN settings
+ space_type = getattr(self, "space_type", "l2")
+ ef_construction = getattr(self, "ef_construction", 512)
+ m = getattr(self, "m", 16)
+
+ mapping = self._default_text_mapping(
+ dim=dim,
+ engine=engine,
+ space_type=space_type,
+ ef_construction=ef_construction,
+ m=m,
+ vector_field=dynamic_field_name, # Use dynamic field name
+ )
- # Ensure the dynamic field exists in the index
- self._ensure_embedding_field_mapping(
- client=client,
- index_name=self.index_name,
- field_name=dynamic_field_name,
- dim=dim,
- engine=engine,
- space_type=space_type,
- ef_construction=ef_construction,
- m=m,
- )
+ # Ensure index exists with baseline mapping (index.knn: true is required for vector search)
+ index_exists = True
+ try:
+ index_exists = bool(client.indices.exists(index=self.index_name))
+ except OpenSearchException as exists_error:
+ self._log_index_admin_skip("indices.exists", exists_error)
- self.log(f"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...")
+ try:
+ if not index_exists:
+ self.log(f"Creating index '{self.index_name}' with base mapping")
+ client.indices.create(index=self.index_name, body=mapping)
+ except RequestError as creation_error:
+ if creation_error.error == "resource_already_exists_exception":
+ pass # Index was created concurrently
+ else:
+ error_msg = str(creation_error).lower()
+ if "invalid engine" in error_msg or "illegal_argument" in error_msg:
+ if "jvector" in error_msg:
+ msg = (
+ "The 'jvector' engine is not available in your OpenSearch installation. "
+ "Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+."
+ )
+ raise ValueError(msg) from creation_error
+ if "index.knn" in error_msg:
+ msg = (
+ "The index has index.knn: false. Delete the existing index and let the "
+ "component recreate it, or create a new index with a different name."
+ )
+ raise ValueError(msg) from creation_error
+ logger.warning(f"Failed to create index '{self.index_name}': {creation_error}")
+ raise
+
+ # Ensure the dynamic field exists in the index
+ self._ensure_embedding_field_mapping(
+ client=client,
+ index_name=self.index_name,
+ field_name=dynamic_field_name,
+ dim=dim,
+ engine=engine,
+ space_type=space_type,
+ ef_construction=ef_construction,
+ m=m,
+ )
+
+ self.log(
+ f"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'..."
+ )
logger.info(f"Will store embeddings in field: {dynamic_field_name}")
logger.info(f"Will tag documents with embedding_model: {embedding_model}")
@@ -1362,7 +1689,11 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
raw = [raw]
explicit_clauses: list[dict] = []
for f in raw or []:
- if "term" in f and isinstance(f["term"], dict) and not self._is_placeholder_term(f["term"]):
+ if (
+ "term" in f
+ and isinstance(f["term"], dict)
+ and not self._is_placeholder_term(f["term"])
+ ):
explicit_clauses.append(f)
elif "terms" in f and isinstance(f["terms"], dict):
field, vals = next(iter(f["terms"].items()))
@@ -1391,6 +1722,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
context_clauses.append({"terms": {field: values}})
return context_clauses
+
def _parse_filter_expression(self) -> dict | None:
"""Parse and validate optional filter_expression JSON.
@@ -1446,6 +1778,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
return float(score_threshold)
def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:
+
"""Detect which embedding models have documents in the index.
Uses aggregation to find all unique embedding_model values, optionally
@@ -1459,7 +1792,10 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
List of embedding model names found in the index
"""
try:
- agg_query = {"size": 0, "aggs": {"embedding_models": {"terms": {"field": "embedding_model", "size": 10}}}}
+ agg_query = {
+ "size": 0,
+ "aggs": {"embedding_models": {"terms": {"field": "embedding_model", "size": 10}}},
+ }
# Apply filters to model detection if any exist
if filter_clauses:
@@ -1481,7 +1817,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
)
if not models:
total_hits = result.get("hits", {}).get("total", {})
- total_count = total_hits.get("value", 0) if isinstance(total_hits, dict) else total_hits
+ total_count = (
+ total_hits.get("value", 0) if isinstance(total_hits, dict) else total_hits
+ )
logger.warning(
f"No embedding_model values found in index '{self.index_name}'. "
f"Total docs in index: {total_count}. "
@@ -1518,7 +1856,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
if not field_name:
return False
if properties is None:
- logger.warning(f"Mapping metadata unavailable; assuming field '{field_name}' is usable.")
+ logger.warning(
+ f"Mapping metadata unavailable; assuming field '{field_name}' is usable."
+ )
return True
field_def = properties.get(field_name)
if not isinstance(field_def, dict):
@@ -1529,7 +1869,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
nested_props = field_def.get("properties")
return bool(isinstance(nested_props, dict) and nested_props.get("type") == "knn_vector")
- def _get_field_dimension(self, properties: dict[str, Any] | None, field_name: str) -> int | None:
+ def _get_field_dimension(
+ self, properties: dict[str, Any] | None, field_name: str
+ ) -> int | None:
"""Get the dimension of a knn_vector field from the index mapping.
Args:
@@ -1617,7 +1959,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
raise ValueError(msg)
# Check if embedding is None (fail-safe mode)
- if self.embedding is None or (isinstance(self.embedding, list) and all(e is None for e in self.embedding)):
+ if self.embedding is None or (
+ isinstance(self.embedding, list) and all(e is None for e in self.embedding)
+ ):
logger.error("Embedding returned None (fail-safe mode enabled). Cannot perform search.")
return []
@@ -1679,13 +2023,17 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
if model_str not in embedding_by_model:
# Use the dedicated embedding instance from the dict
embedding_by_model[model_str] = dedicated_embedding
- logger.info(f"Mapped available model '{model_str}' to dedicated embedding instance")
+ logger.info(
+ f"Mapped available model '{model_str}' to dedicated embedding instance"
+ )
else:
# Conflict detected - track it
if model_str not in identifier_conflicts:
identifier_conflicts[model_str] = [embedding_by_model[model_str]]
identifier_conflicts[model_str].append(dedicated_embedding)
- logger.warning(f"Available model '{model_str}' has conflict - used by multiple embeddings")
+ logger.warning(
+ f"Available model '{model_str}' has conflict - used by multiple embeddings"
+ )
# Also map traditional identifiers (for backward compatibility)
if deployment:
@@ -1707,7 +2055,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
if identifier not in identifier_conflicts:
identifier_conflicts[identifier] = [embedding_by_model[identifier]]
identifier_conflicts[identifier].append(emb_obj)
- logger.warning(f"Identifier '{identifier}' has conflict - used by multiple embeddings")
+ logger.warning(
+ f"Identifier '{identifier}' has conflict - used by multiple embeddings"
+ )
# For embeddings with model+deployment, create combined identifier
# This helps when deployment is the same but model differs
@@ -1715,7 +2065,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
combined_id = f"{deployment}:{model}"
if combined_id not in embedding_by_model:
embedding_by_model[combined_id] = emb_obj
- logger.info(f"Created combined identifier '{combined_id}' for embedding object {idx}")
+ logger.info(
+ f"Created combined identifier '{combined_id}' for embedding object {idx}"
+ )
# Log conflicts
if identifier_conflicts:
@@ -1724,7 +2076,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
f"Consider using combined format 'deployment:model' or specifying unique model names."
)
for conflict_id, emb_list in identifier_conflicts.items():
- logger.warning(f" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier")
+ logger.warning(
+ f" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier"
+ )
logger.info(f"Generating embeddings for {len(available_models)} models in index")
logger.info(f"Available embedding identifiers: {list(embedding_by_model.keys())}")
@@ -1764,7 +2118,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
vec = emb_obj.embed_query(q)
query_embeddings[model_name] = vec
matched_models.append(model_name)
- logger.info(f"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})")
+ logger.info(
+ f"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})"
+ )
self.log(f"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding")
else:
# No matching embedding found for this model
@@ -1773,14 +2129,27 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
f"No matching embedding found for model '{model_name}'. "
f"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}"
)
- self.log(f"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}")
- except (RuntimeError, ValueError, ConnectionError, TimeoutError, AttributeError, KeyError) as e:
+ self.log(
+ f"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}"
+ )
+ except (
+ RuntimeError,
+ ValueError,
+ ConnectionError,
+ TimeoutError,
+ AttributeError,
+ KeyError,
+ ) as e:
logger.warning(f"Failed to generate embedding for {model_name}: {e}")
self.log(f"[ERROR] Embedding generation failed for '{model_name}': {e}")
# Log summary of model matching
- logger.info(f"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched")
- self.log(f"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched")
+ logger.info(
+ f"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched"
+ )
+ self.log(
+ f"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched"
+ )
if unmatched_models:
self.log(f"[WARN] Unmatched models in index: {unmatched_models}")
@@ -1820,7 +2189,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
f"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. "
f"Documents must be indexed with this embedding model before querying."
)
- self.log(f"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'")
+ self.log(
+ f"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'"
+ )
continue
# Validate vector dimensions match the field dimensions
@@ -1831,7 +2202,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
f"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. "
f"Skipping this model to prevent search errors."
)
- self.log(f"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping")
+ self.log(
+ f"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping"
+ )
continue
logger.info(
@@ -1877,7 +2250,10 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
# Build exists filter - document must have at least one embedding field
exists_any_embedding = {
- "bool": {"should": [{"exists": {"field": f}} for f in set(embedding_fields)], "minimum_should_match": 1}
+ "bool": {
+ "should": [{"exists": {"field": f}} for f in set(embedding_fields)],
+ "minimum_should_match": 1,
+ }
}
# Combine user filters with exists filter
@@ -1932,6 +2308,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
"embedding_model",
"allowed_users",
"allowed_groups",
+ "allowed_principals",
],
"size": limit,
}
@@ -1943,7 +2320,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
f"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: "
f"{list(query_embeddings.keys())}"
)
- self.log(f"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}")
+ self.log(
+ f"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}"
+ )
self.log(f"[EXEC] Embedding models used: {list(query_embeddings.keys())}")
self.log(f"[EXEC] KNN fields being queried: {embedding_fields}")
@@ -1959,7 +2338,9 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
)
fallback_body = copy.deepcopy(body)
try:
- fallback_body["query"]["bool"]["should"][0]["dis_max"]["queries"] = knn_queries_without_candidates
+ fallback_body["query"]["bool"]["should"][0]["dis_max"]["queries"] = (
+ knn_queries_without_candidates
+ )
except (KeyError, IndexError, TypeError) as inner_err:
raise e from inner_err
resp = client.search(
@@ -2020,6 +2401,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
]
def search_documents(self) -> Table:
+
"""Search documents and return results as a Table.
This is the main interface method that performs the multi-model search using the
@@ -2035,6 +2417,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
Raises:
Exception: If search operation fails
"""
+
try:
# Always build/cache the vector store to ensure ingestion happens
logger.info(f"Search query: {self.search_query}")
@@ -2045,17 +2428,22 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
search_query = (self.search_query or "").strip()
if not search_query:
self.log("No search query provided - ingestion completed, returning empty results")
- return Table()
+
+ return Table(data=[])
# Perform search with the provided query
raw = self.search(search_query)
- return Table([Data(text=hit["page_content"], **hit["metadata"]) for hit in raw])
+ raw_list = [Data(text=hit["page_content"], **hit["metadata"]) for hit in raw]
+ return Table(data=raw_list)
+
except Exception as e:
self.log(f"search_documents error: {e}")
raise
# -------- dynamic UI handling (auth switch) --------
- async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:
+ async def update_build_config(
+ self, build_config: dict, field_value: str, field_name: str | None = None
+ ) -> dict:
"""Dynamically update component configuration based on field changes.
This method handles real-time UI updates, particularly for authentication
@@ -2074,6 +2462,7 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
mode = (field_value or "basic").strip().lower()
is_basic = mode == "basic"
is_jwt = mode == "jwt"
+ is_openrag = mode == "openrag"
build_config["username"]["show"] = is_basic
build_config["password"]["show"] = is_basic
@@ -2085,11 +2474,25 @@ class OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreCompon
build_config["username"]["required"] = is_basic
build_config["password"]["required"] = is_basic
- build_config["jwt_token"]["required"] = is_jwt
+ # build_config["jwt_token"]["required"] = is_jwt
build_config["jwt_header"]["required"] = is_jwt
build_config["bearer_prefix"]["required"] = False
- if is_basic:
+ # In 'openrag' mode, expose the OPENRAG_* fields up front
+ # since they are the only credentials required.
+ for openrag_field in (
+ "openrag_ingest_url",
+ "openrag_ingest_token",
+ "openrag_ingest_run_id",
+ "openrag_ingest_batch_size",
+ ):
+ if openrag_field in build_config:
+ build_config[openrag_field]["advanced"] = not is_openrag
+ build_config[openrag_field]["required"] = (
+ is_openrag and openrag_field != "openrag_ingest_batch_size"
+ )
+
+ if is_basic or is_openrag:
build_config["jwt_token"]["value"] = ""
return build_config
diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json
index 056e9ecbe..0d05863e3 100644
--- a/flows/ingestion_flow.json
+++ b/flows/ingestion_flow.json
@@ -525,6 +525,93 @@
"sourceHandle": "{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}",
"target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4",
"targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œDataœ,œDataFrameœ,œTableœ],œtypeœ:œotherœ}"
+ },
+ {
+ "animated": false,
+ "className": "",
+ "data": {
+ "sourceHandle": {
+ "dataType": "TextInput",
+ "id": "TextInput-OpenRAGIngest-url",
+ "name": "text",
+ "output_types": [
+ "Message"
+ ]
+ },
+ "targetHandle": {
+ "fieldName": "openrag_ingest_url",
+ "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4",
+ "inputTypes": [
+ "Text",
+ "Message"
+ ],
+ "type": "str"
+ }
+ },
+ "id": "xy-edge__TextInput-OpenRAGIngest-url{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-urlœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4{œfieldNameœ:œopenrag_ingest_urlœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}",
+ "selected": false,
+ "source": "TextInput-OpenRAGIngest-url",
+ "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-urlœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}",
+ "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4",
+ "targetHandle": "{œfieldNameœ:œopenrag_ingest_urlœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}"
+ },
+ {
+ "animated": false,
+ "className": "",
+ "data": {
+ "sourceHandle": {
+ "dataType": "TextInput",
+ "id": "TextInput-OpenRAGIngest-token",
+ "name": "text",
+ "output_types": [
+ "Message"
+ ]
+ },
+ "targetHandle": {
+ "fieldName": "openrag_ingest_token",
+ "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4",
+ "inputTypes": [
+ "Text",
+ "Message"
+ ],
+ "type": "str"
+ }
+ },
+ "id": "xy-edge__TextInput-OpenRAGIngest-token{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-tokenœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4{œfieldNameœ:œopenrag_ingest_tokenœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}",
+ "selected": false,
+ "source": "TextInput-OpenRAGIngest-token",
+ "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-tokenœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}",
+ "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4",
+ "targetHandle": "{œfieldNameœ:œopenrag_ingest_tokenœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}"
+ },
+ {
+ "animated": false,
+ "className": "",
+ "data": {
+ "sourceHandle": {
+ "dataType": "TextInput",
+ "id": "TextInput-OpenRAGIngest-run_id",
+ "name": "text",
+ "output_types": [
+ "Message"
+ ]
+ },
+ "targetHandle": {
+ "fieldName": "openrag_ingest_run_id",
+ "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4",
+ "inputTypes": [
+ "Text",
+ "Message"
+ ],
+ "type": "str"
+ }
+ },
+ "id": "xy-edge__TextInput-OpenRAGIngest-run_id{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-run_idœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4{œfieldNameœ:œopenrag_ingest_run_idœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}",
+ "selected": false,
+ "source": "TextInput-OpenRAGIngest-run_id",
+ "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-run_idœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}",
+ "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4",
+ "targetHandle": "{œfieldNameœ:œopenrag_ingest_run_idœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4œ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}"
}
],
"nodes": [
@@ -4025,7 +4112,7 @@
"show": true,
"title_case": false,
"type": "code",
- "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import Table\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None}\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"jwt\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list) and len(val) > min_vector_length and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\")\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n return return_ids\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\")\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\")\n self.log(\"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\")\n return\n\n logger.debug(f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\")\n self.log(f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\")\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name and self.embedding_model_name.strip():\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\")\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\")\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\")\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {executor.submit(embed_chunk_with_retry, chunk, idx): idx for idx, chunk in enumerate(texts)}\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n try:\n if not client.indices.exists(index=self.index_name):\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\")\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\"size\": 0, \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}}}\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\")\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(self, properties: dict[str, Any] | None, field_name: str) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (isinstance(self.embedding, list) and all(e is None for e in self.embedding)):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(f\"Mapped available model '{model_str}' to dedicated embedding instance\")\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(f\"Available model '{model_str}' has conflict - used by multiple embeddings\")\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(f\"Identifier '{identifier}' has conflict - used by multiple embeddings\")\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(f\"Created combined identifier '{combined_id}' for embedding object {idx}\")\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\")\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\")\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\")\n except (RuntimeError, ValueError, ConnectionError, TimeoutError, AttributeError, KeyError) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n self.log(f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\")\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\")\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)], \"minimum_should_match\": 1}\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\")\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = knn_queries_without_candidates\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> Table:\n \"\"\"Search documents and return results as a Table.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Table (DataFrame) format\n so downstream Parser components can consume them directly.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n Table containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n return Table()\n\n # Perform search with the provided query\n raw = self.search(search_query)\n return Table([Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw])\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n if is_basic:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"
+ "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import Table\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None}\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n StrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n masked_token = (\n f\"{token[:4]}...{token[-4:]}\" if len(token) >= 8 else (\"<set>\" if token else \"\")\n )\n debug_payload = {\n \"openrag_ingest_url\": url,\n \"openrag_ingest_url_len\": len(url),\n \"openrag_ingest_token_masked\": masked_token,\n \"openrag_ingest_token_len\": len(token),\n \"openrag_ingest_run_id\": ingest_run_id,\n \"raw_url_type\": type(self.openrag_ingest_url).__name__,\n \"raw_token_type\": type(self.openrag_ingest_token).__name__,\n \"raw_run_id_type\": type(self.openrag_ingest_run_id).__name__,\n }\n logger.warning(f\"[OpenRAG callback config] {debug_payload}\")\n try:\n self.log(f\"[OpenRAG callback config] {debug_payload}\")\n except Exception:\n pass\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n masked_token = (\n f\"{token[:4]}...{token[-4:]}\" if len(token) >= 8 else (\"<set>\" if token else \"\")\n )\n request_summary = {\n \"url\": url,\n \"ingest_run_id\": ingest_run_id,\n \"token_masked\": masked_token,\n \"total_chunks\": len(requests),\n \"batch_size\": batch_size,\n \"timeout_s\": timeout,\n }\n logger.warning(f\"[OpenRAG ingest POST] {request_summary}\")\n try:\n self.log(f\"[OpenRAG ingest POST] {request_summary}\")\n except Exception:\n pass\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.warning(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"url={url} chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n response_summary = {\n \"batch\": batch_number,\n \"url\": url,\n \"status\": response.status_code,\n \"final_url\": str(response.request.url),\n \"response_headers\": dict(response.headers),\n \"body_preview\": response.text[:500],\n }\n logger.warning(f\"[OpenRAG ingest POST resp] {response_summary}\")\n try:\n self.log(f\"[OpenRAG ingest POST resp] {response_summary}\")\n except Exception:\n pass\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code}, \"\n f\"url={url}): {response.text[:1000]}\"\n )\n raise RuntimeError(msg)\n\n self.log(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", self.openrag_ingest_url),\n (\"openrag_ingest_token\", self.openrag_ingest_token),\n (\"openrag_ingest_run_id\", self.openrag_ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\n \"size\": 0,\n \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}},\n }\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (\n isinstance(self.embedding, list) and all(e is None for e in self.embedding)\n ):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except (\n RuntimeError,\n ValueError,\n ConnectionError,\n TimeoutError,\n AttributeError,\n KeyError,\n ) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\n \"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)],\n \"minimum_should_match\": 1,\n }\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> Table:\n\n \"\"\"Search documents and return results as a Table.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Table (DataFrame) format\n so downstream Parser components can consume them directly.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n Table containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return Table(data=[])\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return Table(data=raw_list)\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"
},
"docs_metadata": {
"_input_type": "TableInput",
@@ -4575,6 +4662,99 @@
"track_in_telemetry": true,
"type": "bool",
"value": false
+ },
+ "openrag_ingest_url": {
+ "_input_type": "StrInput",
+ "advanced": true,
+ "display_name": "OpenRAG Ingest URL",
+ "dynamic": false,
+ "info": "Internal OpenRAG callback URL for backend-owned document indexing.",
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": true,
+ "name": "openrag_ingest_url",
+ "override_skip": false,
+ "placeholder": "",
+ "required": false,
+ "show": false,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_URL",
+ "input_types": [
+ "Text",
+ "Message"
+ ]
+ },
+ "openrag_ingest_token": {
+ "_input_type": "StrInput",
+ "advanced": true,
+ "display_name": "OpenRAG Ingest Token",
+ "dynamic": false,
+ "info": "Short-lived token used only for OpenRAG ingest callbacks.",
+ "input_types": [
+ "Text",
+ "Message"
+ ],
+ "load_from_db": true,
+ "name": "openrag_ingest_token",
+ "override_skip": false,
+ "password": true,
+ "placeholder": "",
+ "required": false,
+ "show": false,
+ "title_case": false,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_TOKEN"
+ },
+ "openrag_ingest_run_id": {
+ "_input_type": "StrInput",
+ "advanced": true,
+ "display_name": "OpenRAG Ingest Run ID",
+ "dynamic": false,
+ "info": "",
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": true,
+ "name": "openrag_ingest_run_id",
+ "override_skip": false,
+ "placeholder": "",
+ "required": false,
+ "show": false,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_RUN_ID",
+ "input_types": [
+ "Text",
+ "Message"
+ ]
+ },
+ "openrag_ingest_batch_size": {
+ "_input_type": "IntInput",
+ "advanced": true,
+ "display_name": "OpenRAG Ingest Batch Size",
+ "dynamic": false,
+ "info": "",
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": false,
+ "name": "openrag_ingest_batch_size",
+ "override_skip": false,
+ "placeholder": "",
+ "required": false,
+ "show": false,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "int",
+ "value": 100
}
},
"tool_mode": false
@@ -7401,6 +7581,465 @@
},
"selected": false,
"type": "genericNode"
+ },
+ {
+ "data": {
+ "id": "TextInput-OpenRAGIngest-url",
+ "node": {
+ "base_classes": [
+ "Message"
+ ],
+ "beta": false,
+ "conditional_paths": [],
+ "custom_fields": {},
+ "description": "Get user text inputs.",
+ "display_name": "Text Input",
+ "documentation": "https://docs.langflow.org/text-input-and-output",
+ "edited": false,
+ "field_order": [
+ "input_value",
+ "use_global_variable"
+ ],
+ "frozen": false,
+ "icon": "type",
+ "last_updated": "2026-02-27T18:37:07.463Z",
+ "legacy": false,
+ "metadata": {
+ "code_hash": "518f16485886",
+ "dependencies": {
+ "dependencies": [
+ {
+ "name": "lfx",
+ "version": null
+ }
+ ],
+ "total_dependencies": 1
+ },
+ "module": "lfx.components.input_output.text.TextInputComponent"
+ },
+ "minimized": false,
+ "output_types": [],
+ "outputs": [
+ {
+ "allows_loop": false,
+ "cache": true,
+ "display_name": "Output Text",
+ "group_outputs": false,
+ "loop_types": null,
+ "method": "text_response",
+ "name": "text",
+ "options": null,
+ "required_inputs": null,
+ "selected": "Message",
+ "tool_mode": true,
+ "types": [
+ "Message"
+ ],
+ "value": "__UNDEFINED__"
+ }
+ ],
+ "pinned": false,
+ "template": {
+ "_frontend_node_flow_id": {
+ "value": "5488df7c-b93f-4f87-a446-b67028bc0813"
+ },
+ "_frontend_node_folder_id": {
+ "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e"
+ },
+ "_type": "Component",
+ "code": {
+ "advanced": true,
+ "dynamic": true,
+ "fileTypes": [],
+ "file_path": "",
+ "info": "",
+ "list": false,
+ "load_from_db": false,
+ "multiline": true,
+ "name": "code",
+ "password": false,
+ "placeholder": "",
+ "required": true,
+ "show": true,
+ "title_case": false,
+ "type": "code",
+ "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n"
+ },
+ "input_value": {
+ "_input_type": "MultilineInput",
+ "advanced": false,
+ "ai_enabled": false,
+ "copy_field": false,
+ "display_name": "Text",
+ "dynamic": false,
+ "info": "Text to be passed as input.",
+ "input_types": [
+ "Message"
+ ],
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": true,
+ "multiline": false,
+ "name": "input_value",
+ "override_skip": false,
+ "password": true,
+ "placeholder": "",
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_input": true,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_URL"
+ },
+ "is_refresh": false,
+ "use_global_variable": {
+ "_input_type": "BoolInput",
+ "advanced": true,
+ "display_name": "Use Global Variable",
+ "dynamic": false,
+ "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.",
+ "list": false,
+ "list_add_label": "Add More",
+ "name": "use_global_variable",
+ "override_skip": false,
+ "placeholder": "",
+ "real_time_refresh": true,
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": true,
+ "type": "bool",
+ "value": true
+ }
+ },
+ "tool_mode": false
+ },
+ "showNode": false,
+ "type": "TextInput"
+ },
+ "dragging": false,
+ "id": "TextInput-OpenRAGIngest-url",
+ "measured": {
+ "height": 52,
+ "width": 192
+ },
+ "position": {
+ "x": 1036.949537574781,
+ "y": 1280.6023642029047
+ },
+ "selected": false,
+ "type": "genericNode"
+ },
+ {
+ "data": {
+ "id": "TextInput-OpenRAGIngest-token",
+ "node": {
+ "base_classes": [
+ "Message"
+ ],
+ "beta": false,
+ "conditional_paths": [],
+ "custom_fields": {},
+ "description": "Get user text inputs.",
+ "display_name": "Text Input",
+ "documentation": "https://docs.langflow.org/text-input-and-output",
+ "edited": false,
+ "field_order": [
+ "input_value",
+ "use_global_variable"
+ ],
+ "frozen": false,
+ "icon": "type",
+ "last_updated": "2026-02-27T18:37:07.463Z",
+ "legacy": false,
+ "metadata": {
+ "code_hash": "518f16485886",
+ "dependencies": {
+ "dependencies": [
+ {
+ "name": "lfx",
+ "version": null
+ }
+ ],
+ "total_dependencies": 1
+ },
+ "module": "lfx.components.input_output.text.TextInputComponent"
+ },
+ "minimized": false,
+ "output_types": [],
+ "outputs": [
+ {
+ "allows_loop": false,
+ "cache": true,
+ "display_name": "Output Text",
+ "group_outputs": false,
+ "loop_types": null,
+ "method": "text_response",
+ "name": "text",
+ "options": null,
+ "required_inputs": null,
+ "selected": "Message",
+ "tool_mode": true,
+ "types": [
+ "Message"
+ ],
+ "value": "__UNDEFINED__"
+ }
+ ],
+ "pinned": false,
+ "template": {
+ "_frontend_node_flow_id": {
+ "value": "5488df7c-b93f-4f87-a446-b67028bc0813"
+ },
+ "_frontend_node_folder_id": {
+ "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e"
+ },
+ "_type": "Component",
+ "code": {
+ "advanced": true,
+ "dynamic": true,
+ "fileTypes": [],
+ "file_path": "",
+ "info": "",
+ "list": false,
+ "load_from_db": false,
+ "multiline": true,
+ "name": "code",
+ "password": false,
+ "placeholder": "",
+ "required": true,
+ "show": true,
+ "title_case": false,
+ "type": "code",
+ "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n"
+ },
+ "input_value": {
+ "_input_type": "MultilineInput",
+ "advanced": false,
+ "ai_enabled": false,
+ "copy_field": false,
+ "display_name": "Text",
+ "dynamic": false,
+ "info": "Text to be passed as input.",
+ "input_types": [
+ "Message"
+ ],
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": true,
+ "multiline": false,
+ "name": "input_value",
+ "override_skip": false,
+ "password": true,
+ "placeholder": "",
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_input": true,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_TOKEN"
+ },
+ "is_refresh": false,
+ "use_global_variable": {
+ "_input_type": "BoolInput",
+ "advanced": true,
+ "display_name": "Use Global Variable",
+ "dynamic": false,
+ "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.",
+ "list": false,
+ "list_add_label": "Add More",
+ "name": "use_global_variable",
+ "override_skip": false,
+ "placeholder": "",
+ "real_time_refresh": true,
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": true,
+ "type": "bool",
+ "value": true
+ }
+ },
+ "tool_mode": false
+ },
+ "showNode": false,
+ "type": "TextInput"
+ },
+ "dragging": false,
+ "id": "TextInput-OpenRAGIngest-token",
+ "measured": {
+ "height": 52,
+ "width": 192
+ },
+ "position": {
+ "x": 1036.949537574781,
+ "y": 1362.6023642029047
+ },
+ "selected": false,
+ "type": "genericNode"
+ },
+ {
+ "data": {
+ "id": "TextInput-OpenRAGIngest-run_id",
+ "node": {
+ "base_classes": [
+ "Message"
+ ],
+ "beta": false,
+ "conditional_paths": [],
+ "custom_fields": {},
+ "description": "Get user text inputs.",
+ "display_name": "Text Input",
+ "documentation": "https://docs.langflow.org/text-input-and-output",
+ "edited": false,
+ "field_order": [
+ "input_value",
+ "use_global_variable"
+ ],
+ "frozen": false,
+ "icon": "type",
+ "last_updated": "2026-02-27T18:37:07.463Z",
+ "legacy": false,
+ "metadata": {
+ "code_hash": "518f16485886",
+ "dependencies": {
+ "dependencies": [
+ {
+ "name": "lfx",
+ "version": null
+ }
+ ],
+ "total_dependencies": 1
+ },
+ "module": "lfx.components.input_output.text.TextInputComponent"
+ },
+ "minimized": false,
+ "output_types": [],
+ "outputs": [
+ {
+ "allows_loop": false,
+ "cache": true,
+ "display_name": "Output Text",
+ "group_outputs": false,
+ "loop_types": null,
+ "method": "text_response",
+ "name": "text",
+ "options": null,
+ "required_inputs": null,
+ "selected": "Message",
+ "tool_mode": true,
+ "types": [
+ "Message"
+ ],
+ "value": "__UNDEFINED__"
+ }
+ ],
+ "pinned": false,
+ "template": {
+ "_frontend_node_flow_id": {
+ "value": "5488df7c-b93f-4f87-a446-b67028bc0813"
+ },
+ "_frontend_node_folder_id": {
+ "value": "2bef1fdd-4d60-4bb6-8fd2-c0a3eae09d1e"
+ },
+ "_type": "Component",
+ "code": {
+ "advanced": true,
+ "dynamic": true,
+ "fileTypes": [],
+ "file_path": "",
+ "info": "",
+ "list": false,
+ "load_from_db": false,
+ "multiline": true,
+ "name": "code",
+ "password": false,
+ "placeholder": "",
+ "required": true,
+ "show": true,
+ "title_case": false,
+ "type": "code",
+ "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n"
+ },
+ "input_value": {
+ "_input_type": "MultilineInput",
+ "advanced": false,
+ "ai_enabled": false,
+ "copy_field": false,
+ "display_name": "Text",
+ "dynamic": false,
+ "info": "Text to be passed as input.",
+ "input_types": [
+ "Message"
+ ],
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": true,
+ "multiline": false,
+ "name": "input_value",
+ "override_skip": false,
+ "password": true,
+ "placeholder": "",
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_input": true,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_RUN_ID"
+ },
+ "is_refresh": false,
+ "use_global_variable": {
+ "_input_type": "BoolInput",
+ "advanced": true,
+ "display_name": "Use Global Variable",
+ "dynamic": false,
+ "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.",
+ "list": false,
+ "list_add_label": "Add More",
+ "name": "use_global_variable",
+ "override_skip": false,
+ "placeholder": "",
+ "real_time_refresh": true,
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": true,
+ "type": "bool",
+ "value": true
+ }
+ },
+ "tool_mode": false
+ },
+ "showNode": false,
+ "type": "TextInput"
+ },
+ "dragging": false,
+ "id": "TextInput-OpenRAGIngest-run_id",
+ "measured": {
+ "height": 52,
+ "width": 192
+ },
+ "position": {
+ "x": 1036.949537574781,
+ "y": 1444.6023642029047
+ },
+ "selected": false,
+ "type": "genericNode"
}
],
"viewport": {
diff --git a/flows/openrag_agent.json b/flows/openrag_agent.json
index ccd96c838..faebd4249 100644
--- a/flows/openrag_agent.json
+++ b/flows/openrag_agent.json
@@ -2738,7 +2738,7 @@
"show": true,
"title_case": false,
"type": "code",
- "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import Table\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None}\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"jwt\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list) and len(val) > min_vector_length and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\")\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n return return_ids\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\")\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\")\n self.log(\"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\")\n return\n\n logger.debug(f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\")\n self.log(f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\")\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name and self.embedding_model_name.strip():\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\")\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\")\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\")\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {executor.submit(embed_chunk_with_retry, chunk, idx): idx for idx, chunk in enumerate(texts)}\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n try:\n if not client.indices.exists(index=self.index_name):\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\")\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\"size\": 0, \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}}}\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\")\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(self, properties: dict[str, Any] | None, field_name: str) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (isinstance(self.embedding, list) and all(e is None for e in self.embedding)):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(f\"Mapped available model '{model_str}' to dedicated embedding instance\")\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(f\"Available model '{model_str}' has conflict - used by multiple embeddings\")\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(f\"Identifier '{identifier}' has conflict - used by multiple embeddings\")\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(f\"Created combined identifier '{combined_id}' for embedding object {idx}\")\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\")\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\")\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\")\n except (RuntimeError, ValueError, ConnectionError, TimeoutError, AttributeError, KeyError) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n self.log(f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\")\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\")\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)], \"minimum_should_match\": 1}\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\")\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = knn_queries_without_candidates\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> Table:\n \"\"\"Search documents and return results as a Table.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Table (DataFrame) format\n so downstream Parser components can consume them directly.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n Table containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n return Table()\n\n # Perform search with the provided query\n raw = self.search(search_query)\n return Table([Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw])\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n if is_basic:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"
+ "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import Table\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None}\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n StrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n masked_token = (\n f\"{token[:4]}...{token[-4:]}\" if len(token) >= 8 else (\"<set>\" if token else \"\")\n )\n debug_payload = {\n \"openrag_ingest_url\": url,\n \"openrag_ingest_url_len\": len(url),\n \"openrag_ingest_token_masked\": masked_token,\n \"openrag_ingest_token_len\": len(token),\n \"openrag_ingest_run_id\": ingest_run_id,\n \"raw_url_type\": type(self.openrag_ingest_url).__name__,\n \"raw_token_type\": type(self.openrag_ingest_token).__name__,\n \"raw_run_id_type\": type(self.openrag_ingest_run_id).__name__,\n }\n logger.warning(f\"[OpenRAG callback config] {debug_payload}\")\n try:\n self.log(f\"[OpenRAG callback config] {debug_payload}\")\n except Exception:\n pass\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n masked_token = (\n f\"{token[:4]}...{token[-4:]}\" if len(token) >= 8 else (\"<set>\" if token else \"\")\n )\n request_summary = {\n \"url\": url,\n \"ingest_run_id\": ingest_run_id,\n \"token_masked\": masked_token,\n \"total_chunks\": len(requests),\n \"batch_size\": batch_size,\n \"timeout_s\": timeout,\n }\n logger.warning(f\"[OpenRAG ingest POST] {request_summary}\")\n try:\n self.log(f\"[OpenRAG ingest POST] {request_summary}\")\n except Exception:\n pass\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.warning(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"url={url} chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n response_summary = {\n \"batch\": batch_number,\n \"url\": url,\n \"status\": response.status_code,\n \"final_url\": str(response.request.url),\n \"response_headers\": dict(response.headers),\n \"body_preview\": response.text[:500],\n }\n logger.warning(f\"[OpenRAG ingest POST resp] {response_summary}\")\n try:\n self.log(f\"[OpenRAG ingest POST resp] {response_summary}\")\n except Exception:\n pass\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code}, \"\n f\"url={url}): {response.text[:1000]}\"\n )\n raise RuntimeError(msg)\n\n self.log(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", self.openrag_ingest_url),\n (\"openrag_ingest_token\", self.openrag_ingest_token),\n (\"openrag_ingest_run_id\", self.openrag_ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\n \"size\": 0,\n \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}},\n }\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (\n isinstance(self.embedding, list) and all(e is None for e in self.embedding)\n ):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except (\n RuntimeError,\n ValueError,\n ConnectionError,\n TimeoutError,\n AttributeError,\n KeyError,\n ) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\n \"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)],\n \"minimum_should_match\": 1,\n }\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> Table:\n\n \"\"\"Search documents and return results as a Table.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Table (DataFrame) format\n so downstream Parser components can consume them directly.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n Table containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return Table(data=[])\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return Table(data=raw_list)\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"
},
"docs_metadata": {
"_input_type": "TableInput",
diff --git a/flows/openrag_nudges.json b/flows/openrag_nudges.json
index 6ab881cb9..764bd39d5 100644
--- a/flows/openrag_nudges.json
+++ b/flows/openrag_nudges.json
@@ -2489,7 +2489,7 @@
"show": true,
"title_case": false,
"type": "code",
- "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import Table\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None}\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"jwt\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list) and len(val) > min_vector_length and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\")\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n return return_ids\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\")\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\")\n self.log(\"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\")\n return\n\n logger.debug(f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\")\n self.log(f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\")\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name and self.embedding_model_name.strip():\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\")\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\")\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\")\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {executor.submit(embed_chunk_with_retry, chunk, idx): idx for idx, chunk in enumerate(texts)}\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n try:\n if not client.indices.exists(index=self.index_name):\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\")\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\"size\": 0, \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}}}\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\")\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(self, properties: dict[str, Any] | None, field_name: str) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (isinstance(self.embedding, list) and all(e is None for e in self.embedding)):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(f\"Mapped available model '{model_str}' to dedicated embedding instance\")\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(f\"Available model '{model_str}' has conflict - used by multiple embeddings\")\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(f\"Identifier '{identifier}' has conflict - used by multiple embeddings\")\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(f\"Created combined identifier '{combined_id}' for embedding object {idx}\")\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\")\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\")\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\")\n except (RuntimeError, ValueError, ConnectionError, TimeoutError, AttributeError, KeyError) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n self.log(f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\")\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\")\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)], \"minimum_should_match\": 1}\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\")\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = knn_queries_without_candidates\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> Table:\n \"\"\"Search documents and return results as a Table.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Table (DataFrame) format\n so downstream Parser components can consume them directly.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n Table containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n return Table()\n\n # Perform search with the provided query\n raw = self.search(search_query)\n return Table([Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw])\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n if is_basic:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"
+ "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import Table\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None}\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n StrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n masked_token = (\n f\"{token[:4]}...{token[-4:]}\" if len(token) >= 8 else (\"<set>\" if token else \"\")\n )\n debug_payload = {\n \"openrag_ingest_url\": url,\n \"openrag_ingest_url_len\": len(url),\n \"openrag_ingest_token_masked\": masked_token,\n \"openrag_ingest_token_len\": len(token),\n \"openrag_ingest_run_id\": ingest_run_id,\n \"raw_url_type\": type(self.openrag_ingest_url).__name__,\n \"raw_token_type\": type(self.openrag_ingest_token).__name__,\n \"raw_run_id_type\": type(self.openrag_ingest_run_id).__name__,\n }\n logger.warning(f\"[OpenRAG callback config] {debug_payload}\")\n try:\n self.log(f\"[OpenRAG callback config] {debug_payload}\")\n except Exception:\n pass\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n masked_token = (\n f\"{token[:4]}...{token[-4:]}\" if len(token) >= 8 else (\"<set>\" if token else \"\")\n )\n request_summary = {\n \"url\": url,\n \"ingest_run_id\": ingest_run_id,\n \"token_masked\": masked_token,\n \"total_chunks\": len(requests),\n \"batch_size\": batch_size,\n \"timeout_s\": timeout,\n }\n logger.warning(f\"[OpenRAG ingest POST] {request_summary}\")\n try:\n self.log(f\"[OpenRAG ingest POST] {request_summary}\")\n except Exception:\n pass\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.warning(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"url={url} chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n response_summary = {\n \"batch\": batch_number,\n \"url\": url,\n \"status\": response.status_code,\n \"final_url\": str(response.request.url),\n \"response_headers\": dict(response.headers),\n \"body_preview\": response.text[:500],\n }\n logger.warning(f\"[OpenRAG ingest POST resp] {response_summary}\")\n try:\n self.log(f\"[OpenRAG ingest POST resp] {response_summary}\")\n except Exception:\n pass\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code}, \"\n f\"url={url}): {response.text[:1000]}\"\n )\n raise RuntimeError(msg)\n\n self.log(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", self.openrag_ingest_url),\n (\"openrag_ingest_token\", self.openrag_ingest_token),\n (\"openrag_ingest_run_id\", self.openrag_ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\n \"size\": 0,\n \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}},\n }\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (\n isinstance(self.embedding, list) and all(e is None for e in self.embedding)\n ):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except (\n RuntimeError,\n ValueError,\n ConnectionError,\n TimeoutError,\n AttributeError,\n KeyError,\n ) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\n \"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)],\n \"minimum_should_match\": 1,\n }\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> Table:\n\n \"\"\"Search documents and return results as a Table.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Table (DataFrame) format\n so downstream Parser components can consume them directly.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n Table containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return Table(data=[])\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return Table(data=raw_list)\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"
},
"docs_metadata": {
"_input_type": "TableInput",
diff --git a/flows/openrag_url_mcp.json b/flows/openrag_url_mcp.json
index c09fb70ea..ec34cd3aa 100644
--- a/flows/openrag_url_mcp.json
+++ b/flows/openrag_url_mcp.json
@@ -526,6 +526,93 @@
"sourceHandle": "{œdataTypeœ:œSplitTextœ,œidœ:œSplitText-QIKhgœ,œnameœ:œdataframeœ,œoutput_typesœ:[œTableœ]}",
"target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV",
"targetHandle": "{œfieldNameœ:œingest_dataœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGVœ,œinputTypesœ:[œDataœ,œDataFrameœ,œTableœ],œtypeœ:œotherœ}"
+ },
+ {
+ "animated": false,
+ "className": "",
+ "data": {
+ "sourceHandle": {
+ "dataType": "TextInput",
+ "id": "TextInput-OpenRAGIngest-url",
+ "name": "text",
+ "output_types": [
+ "Message"
+ ]
+ },
+ "targetHandle": {
+ "fieldName": "openrag_ingest_url",
+ "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV",
+ "inputTypes": [
+ "Text",
+ "Message"
+ ],
+ "type": "str"
+ }
+ },
+ "id": "xy-edge__TextInput-OpenRAGIngest-url{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-urlœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV{œfieldNameœ:œopenrag_ingest_urlœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGVœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}",
+ "selected": false,
+ "source": "TextInput-OpenRAGIngest-url",
+ "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-urlœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}",
+ "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV",
+ "targetHandle": "{œfieldNameœ:œopenrag_ingest_urlœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGVœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}"
+ },
+ {
+ "animated": false,
+ "className": "",
+ "data": {
+ "sourceHandle": {
+ "dataType": "TextInput",
+ "id": "TextInput-OpenRAGIngest-token",
+ "name": "text",
+ "output_types": [
+ "Message"
+ ]
+ },
+ "targetHandle": {
+ "fieldName": "openrag_ingest_token",
+ "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV",
+ "inputTypes": [
+ "Text",
+ "Message"
+ ],
+ "type": "str"
+ }
+ },
+ "id": "xy-edge__TextInput-OpenRAGIngest-token{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-tokenœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV{œfieldNameœ:œopenrag_ingest_tokenœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGVœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}",
+ "selected": false,
+ "source": "TextInput-OpenRAGIngest-token",
+ "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-tokenœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}",
+ "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV",
+ "targetHandle": "{œfieldNameœ:œopenrag_ingest_tokenœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGVœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}"
+ },
+ {
+ "animated": false,
+ "className": "",
+ "data": {
+ "sourceHandle": {
+ "dataType": "TextInput",
+ "id": "TextInput-OpenRAGIngest-run_id",
+ "name": "text",
+ "output_types": [
+ "Message"
+ ]
+ },
+ "targetHandle": {
+ "fieldName": "openrag_ingest_run_id",
+ "id": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV",
+ "inputTypes": [
+ "Text",
+ "Message"
+ ],
+ "type": "str"
+ }
+ },
+ "id": "xy-edge__TextInput-OpenRAGIngest-run_id{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-run_idœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}-OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV{œfieldNameœ:œopenrag_ingest_run_idœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGVœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}",
+ "selected": false,
+ "source": "TextInput-OpenRAGIngest-run_id",
+ "sourceHandle": "{œdataTypeœ:œTextInputœ,œidœ:œTextInput-OpenRAGIngest-run_idœ,œnameœ:œtextœ,œoutput_typesœ:[œMessageœ]}",
+ "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV",
+ "targetHandle": "{œfieldNameœ:œopenrag_ingest_run_idœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGVœ,œinputTypesœ:[œTextœ,œMessageœ],œtypeœ:œstrœ}"
}
],
"nodes": [
@@ -3762,7 +3849,7 @@
"show": true,
"title_case": false,
"type": "code",
- "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import Table\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None}\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n ]\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"jwt\",\n options=[\"basic\", \"jwt\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"or 'jwt' for JSON Web Token (Bearer) authentication.\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list) and len(val) > min_vector_length and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\")\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n _id = ids[i] if ids else str(uuid.uuid4())\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n return return_ids\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\")\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\")\n self.log(\"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\")\n return\n\n logger.debug(f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\")\n self.log(f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\")\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name and self.embedding_model_name.strip():\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\")\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\")\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\")\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {executor.submit(embed_chunk_with_retry, chunk, idx): idx for idx, chunk in enumerate(texts)}\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n engine = getattr(self, \"engine\", \"jvector\")\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n try:\n if not client.indices.exists(index=self.index_name):\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\")\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if \"term\" in f and isinstance(f[\"term\"], dict) and not self._is_placeholder_term(f[\"term\"]):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\"size\": 0, \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}}}\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\")\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(self, properties: dict[str, Any] | None, field_name: str) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (isinstance(self.embedding, list) and all(e is None for e in self.embedding)):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(f\"Mapped available model '{model_str}' to dedicated embedding instance\")\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(f\"Available model '{model_str}' has conflict - used by multiple embeddings\")\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(f\"Identifier '{identifier}' has conflict - used by multiple embeddings\")\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(f\"Created combined identifier '{combined_id}' for embedding object {idx}\")\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\")\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\")\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\")\n except (RuntimeError, ValueError, ConnectionError, TimeoutError, AttributeError, KeyError) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n self.log(f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\")\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\")\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\")\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)], \"minimum_should_match\": 1}\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\")\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = knn_queries_without_candidates\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> Table:\n \"\"\"Search documents and return results as a Table.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Table (DataFrame) format\n so downstream Parser components can consume them directly.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n Table containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n return Table()\n\n # Perform search with the provided query\n raw = self.search(search_query)\n return Table([Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw])\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(self, build_config: dict, field_value: str, field_name: str | None = None) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n if is_basic:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"
+ "value": "from __future__ import annotations\n\nimport copy\nimport json\nimport uuid\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom typing import Any\n\nimport httpx\nfrom lfx.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store\nfrom lfx.base.vectorstores.vector_store_connection_decorator import vector_store_connection\nfrom lfx.io import (\n BoolInput,\n DropdownInput,\n HandleInput,\n IntInput,\n MultilineInput,\n Output,\n SecretStrInput,\n StrInput,\n TableInput,\n)\nfrom lfx.log import logger\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import Table\nfrom opensearchpy import OpenSearch, helpers\nfrom opensearchpy.exceptions import OpenSearchException, RequestError\n\nREQUEST_TIMEOUT = 60\nMAX_RETRIES = 5\n\n# watsonx.ai surfaces rate-limit state via these (mostly non-standard) response\n# headers. The IBM SDK acts on the x-requests-limit-* family directly; we log\n# them on a failed embedding call to aid plan/region tuning.\n_WATSONX_RATE_LIMIT_HEADERS = (\n \"x-requests-limit-rate\",\n \"x-requests-limit-remaining\",\n \"x-requests-limit-reset\",\n \"Retry-After\",\n)\n\n\ndef _log_watsonx_rate_limit_headers(error: Exception) -> None:\n \"\"\"Best-effort diagnostic: log watsonx rate-limit headers from a failed call.\n\n The watsonx SDK raises ``ApiRequestFailure``, which carries the originating\n httpx/requests ``Response`` as ``.response``. On a 429 exhaustion we surface\n the documented rate-limit headers so operators can tune throughput.\n \"\"\"\n try:\n response = getattr(error, \"response\", None)\n headers = getattr(response, \"headers\", None)\n if not headers:\n return\n status = getattr(response, \"status_code\", \"unknown\")\n observed = {h: headers.get(h) for h in _WATSONX_RATE_LIMIT_HEADERS if headers.get(h) is not None}\n if str(status) == \"429\" or observed:\n logger.warning(f\"watsonx rate-limit response (status={status}): {observed}\")\n except Exception as log_error: # never let diagnostics mask the real error\n logger.debug(f\"Could not extract watsonx rate-limit headers: {log_error}\")\n\n\ndef normalize_model_name(model_name: str) -> str:\n \"\"\"Normalize embedding model name for use as field suffix.\n\n Converts model names to valid OpenSearch field names by replacing\n special characters and ensuring alphanumeric format.\n\n Args:\n model_name: Original embedding model name (e.g., \"text-embedding-3-small\")\n\n Returns:\n Normalized field suffix (e.g., \"text_embedding_3_small\")\n \"\"\"\n normalized = model_name.lower()\n # Replace common separators with underscores\n normalized = normalized.replace(\"-\", \"_\").replace(\":\", \"_\").replace(\"/\", \"_\").replace(\".\", \"_\")\n # Remove any non-alphanumeric characters except underscores\n normalized = \"\".join(c if c.isalnum() or c == \"_\" else \"_\" for c in normalized)\n # Remove duplicate underscores\n while \"__\" in normalized:\n normalized = normalized.replace(\"__\", \"_\")\n return normalized.strip(\"_\")\n\n\ndef get_embedding_field_name(model_name: str) -> str:\n \"\"\"Get the dynamic embedding field name for a model.\n\n Args:\n model_name: Embedding model name\n\n Returns:\n Field name in format: chunk_embedding_{normalized_model_name}\n \"\"\"\n logger.info(f\"chunk_embedding_{normalize_model_name(model_name)}\")\n return f\"chunk_embedding_{normalize_model_name(model_name)}\"\n\n\n@vector_store_connection\nclass OpenSearchVectorStoreComponentMultimodalMultiEmbedding(LCVectorStoreComponent):\n \"\"\"OpenSearch Vector Store Component with Multi-Model Hybrid Search Capabilities.\n\n This component provides vector storage and retrieval using OpenSearch, combining semantic\n similarity search (KNN) with keyword-based search for optimal results. It supports:\n - Multiple embedding models per index with dynamic field names\n - Automatic detection and querying of all available embedding models\n - Parallel embedding generation for multi-model search\n - Document ingestion with model tracking\n - Advanced filtering and aggregations\n - Flexible authentication options\n\n Features:\n - Multi-model vector storage with dynamic fields (chunk_embedding_{model_name})\n - Hybrid search combining multiple KNN queries (dis_max) + keyword matching\n - Auto-detection of available models in the index\n - Parallel query embedding generation for all detected models\n - Vector storage with configurable engines (jvector, nmslib, faiss, lucene)\n - Flexible authentication (Basic auth, JWT tokens)\n\n Model Name Resolution:\n - Priority: deployment > model > model_name attributes\n - This ensures correct matching between embedding objects and index fields\n - When multiple embeddings are provided, specify embedding_model_name to select which one to use\n - During search, each detected model in the index is matched to its corresponding embedding object\n \"\"\"\n\n display_name: str = \"OpenSearch (Multi-Model Multi-Embedding)\"\n icon: str = \"OpenSearch\"\n description: str = (\n \"Store and search documents using OpenSearch with multi-model hybrid semantic and keyword search. \"\n \"To search use the tools search_documents and raw_search. \"\n \"Search documents takes a query for vector search, for example\\n\"\n ' {search_query: \"components in openrag\"}'\n )\n\n # Keys we consider baseline\n default_keys: list[str] = [\n \"opensearch_url\",\n \"index_name\",\n *[i.name for i in LCVectorStoreComponent.inputs], # search_query, add_documents, etc.\n \"embedding\",\n \"embedding_model_name\",\n \"vector_field\",\n \"number_of_results\",\n \"auth_mode\",\n \"username\",\n \"password\",\n \"jwt_token\",\n \"jwt_header\",\n \"bearer_prefix\",\n \"use_ssl\",\n \"verify_certs\",\n \"filter_expression\",\n \"engine\",\n \"space_type\",\n \"ef_construction\",\n \"m\",\n \"num_candidates\",\n \"docs_metadata\",\n \"request_timeout\",\n \"max_retries\",\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ]\n _openrag_ingest_global_placeholders = {\n \"openrag_ingest_url\": \"OPENRAG_INGEST_URL\",\n \"openrag_ingest_token\": \"OPENRAG_INGEST_TOKEN\",\n \"openrag_ingest_run_id\": \"OPENRAG_INGEST_RUN_ID\",\n }\n\n inputs = [\n TableInput(\n name=\"docs_metadata\",\n display_name=\"Document Metadata\",\n info=(\n \"Additional metadata key-value pairs to be added to all ingested documents. \"\n \"Useful for tagging documents with source information, categories, or other custom attributes.\"\n ),\n table_schema=[\n {\n \"name\": \"key\",\n \"display_name\": \"Key\",\n \"type\": \"str\",\n \"description\": \"Key name\",\n },\n {\n \"name\": \"value\",\n \"display_name\": \"Value\",\n \"type\": \"str\",\n \"description\": \"Value of the metadata\",\n },\n ],\n value=[],\n input_types=[\"Data\", \"JSON\"],\n ),\n StrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n StrInput(\n name=\"opensearch_url\",\n display_name=\"OpenSearch URL\",\n value=\"http://localhost:9200\",\n info=(\n \"The connection URL for your OpenSearch cluster \"\n \"(e.g., http://localhost:9200 for local development or your cloud endpoint).\"\n ),\n ),\n StrInput(\n name=\"index_name\",\n display_name=\"Index Name\",\n value=\"langflow\",\n info=(\n \"The OpenSearch index name where documents will be stored and searched. \"\n \"Will be created automatically if it doesn't exist.\"\n ),\n ),\n DropdownInput(\n name=\"engine\",\n display_name=\"Vector Engine\",\n options=[\"nmslib\", \"faiss\", \"lucene\", \"jvector\"],\n value=\"jvector\",\n info=(\n \"Vector search engine for similarity calculations. 'nmslib' works with standard \"\n \"OpenSearch. 'jvector' requires OpenSearch 2.9+. 'lucene' requires index.knn: true. \"\n \"Amazon OpenSearch Serverless only supports 'nmslib' or 'faiss'.\"\n ),\n advanced=True,\n ),\n DropdownInput(\n name=\"space_type\",\n display_name=\"Distance Metric\",\n options=[\"l2\", \"l1\", \"cosinesimil\", \"linf\", \"innerproduct\"],\n value=\"l2\",\n info=(\n \"Distance metric for calculating vector similarity. 'l2' (Euclidean) is most common, \"\n \"'cosinesimil' for cosine similarity, 'innerproduct' for dot product.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"ef_construction\",\n display_name=\"EF Construction\",\n value=512,\n info=(\n \"Size of the dynamic candidate list during index construction. \"\n \"Higher values improve recall but increase indexing time and memory usage.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"m\",\n display_name=\"M Parameter\",\n value=16,\n info=(\n \"Number of bidirectional connections for each vector in the HNSW graph. \"\n \"Higher values improve search quality but increase memory usage and indexing time.\"\n ),\n advanced=True,\n ),\n IntInput(\n name=\"num_candidates\",\n display_name=\"Candidate Pool Size\",\n value=1000,\n info=(\n \"Number of approximate neighbors to consider for each KNN query. \"\n \"Some OpenSearch deployments do not support this parameter; set to 0 to disable.\"\n ),\n advanced=True,\n ),\n *LCVectorStoreComponent.inputs, # includes search_query, add_documents, etc.\n HandleInput(\n name=\"embedding\", display_name=\"Embedding\", input_types=[\"Embeddings\"], is_list=True\n ),\n StrInput(\n name=\"embedding_model_name\",\n display_name=\"Embedding Model Name\",\n value=\"\",\n info=(\n \"Name of the embedding model to use for ingestion. This selects which embedding from the list \"\n \"will be used to embed documents. Matches on deployment, model, model_id, or model_name. \"\n \"For duplicate deployments, use combined format: 'deployment:model' \"\n \"(e.g., 'text-embedding-ada-002:text-embedding-3-large'). \"\n \"Leave empty to use the first embedding. Error message will show all available identifiers.\"\n ),\n advanced=False,\n ),\n StrInput(\n name=\"vector_field\",\n display_name=\"Legacy Vector Field Name\",\n value=\"chunk_embedding\",\n advanced=True,\n info=(\n \"Legacy field name for backward compatibility. New documents use dynamic fields \"\n \"(chunk_embedding_{model_name}) based on the embedding_model_name.\"\n ),\n ),\n IntInput(\n name=\"number_of_results\",\n display_name=\"Default Result Limit\",\n value=10,\n advanced=True,\n info=(\n \"Default maximum number of search results to return when no limit is \"\n \"specified in the filter expression.\"\n ),\n ),\n MultilineInput(\n name=\"filter_expression\",\n display_name=\"Search Filters (JSON)\",\n value=\"\",\n info=(\n \"Optional JSON configuration for search filtering, result limits, and score thresholds.\\n\\n\"\n \"Format 1 - Explicit filters:\\n\"\n '{\"filter\": [{\"term\": {\"filename\":\"doc.pdf\"}}, '\n '{\"terms\":{\"owner\":[\"user1\",\"user2\"]}}], \"limit\": 10, \"score_threshold\": 1.6}\\n\\n'\n \"Format 2 - Context-style mapping:\\n\"\n '{\"data_sources\":[\"file.pdf\"], \"document_types\":[\"application/pdf\"], \"owners\":[\"user123\"]}\\n\\n'\n \"Use __IMPOSSIBLE_VALUE__ as placeholder to ignore specific filters.\"\n ),\n ),\n # ----- Auth controls (dynamic) -----\n DropdownInput(\n name=\"auth_mode\",\n display_name=\"Authentication Mode\",\n value=\"openrag\",\n options=[\"basic\", \"jwt\", \"openrag\"],\n info=(\n \"Authentication method: 'basic' for username/password authentication, \"\n \"'jwt' for JSON Web Token (Bearer) authentication, or 'openrag' to \"\n \"delegate writes to the OpenRAG backend ingest callback (no direct \"\n \"OpenSearch credentials required — only OPENRAG_* fields).\"\n ),\n real_time_refresh=True,\n advanced=False,\n ),\n StrInput(\n name=\"username\",\n display_name=\"Username\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"password\",\n display_name=\"OpenSearch Password\",\n value=\"admin\",\n show=True,\n ),\n SecretStrInput(\n name=\"jwt_token\",\n display_name=\"JWT Token\",\n value=\"JWT\",\n load_from_db=False,\n show=False,\n info=(\n \"Valid JSON Web Token for authentication. \"\n \"Will be sent in the Authorization header (with optional 'Bearer ' prefix).\"\n ),\n required=False\n ),\n StrInput(\n name=\"jwt_header\",\n display_name=\"JWT Header Name\",\n value=\"Authorization\",\n show=False,\n advanced=True,\n ),\n BoolInput(\n name=\"bearer_prefix\",\n display_name=\"Prefix 'Bearer '\",\n value=False,\n show=False,\n advanced=True,\n ),\n # ----- TLS -----\n BoolInput(\n name=\"use_ssl\",\n display_name=\"Use SSL/TLS\",\n value=True,\n advanced=True,\n info=\"Enable SSL/TLS encryption for secure connections to OpenSearch.\",\n ),\n BoolInput(\n name=\"verify_certs\",\n display_name=\"Verify SSL Certificates\",\n value=False,\n advanced=True,\n info=(\n \"Verify SSL certificates when connecting. \"\n \"Disable for self-signed certificates in development environments.\"\n ),\n ),\n # ----- Timeout / Retry -----\n StrInput(\n name=\"request_timeout\",\n display_name=\"Request Timeout (seconds)\",\n value=\"60\",\n advanced=True,\n info=(\n \"Time in seconds to wait for a response from OpenSearch. \"\n \"Increase for large bulk ingestion or complex hybrid queries.\"\n ),\n ),\n StrInput(\n name=\"max_retries\",\n display_name=\"Max Retries\",\n value=\"3\",\n advanced=True,\n info=\"Number of retries for failed connections before raising an error.\",\n ),\n StrInput(\n name=\"openrag_ingest_url\",\n display_name=\"OpenRAG Ingest URL\",\n value=\"OPENRAG_INGEST_URL\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Internal OpenRAG callback URL for backend-owned document indexing.\",\n ),\n StrInput(\n name=\"openrag_ingest_token\",\n display_name=\"OpenRAG Ingest Token\",\n value=\"OPENRAG_INGEST_TOKEN\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n info=\"Short-lived token used only for OpenRAG ingest callbacks.\",\n ),\n StrInput(\n name=\"openrag_ingest_run_id\",\n display_name=\"OpenRAG Ingest Run ID\",\n value=\"OPENRAG_INGEST_RUN_ID\",\n load_from_db=True,\n input_types=[\"Text\", \"Message\"],\n advanced=True,\n ),\n IntInput(\n name=\"openrag_ingest_batch_size\",\n display_name=\"OpenRAG Ingest Batch Size\",\n value=100,\n advanced=True,\n ),\n ]\n outputs = [\n Output(\n display_name=\"Search Results\",\n name=\"search_results\",\n method=\"search_documents\",\n ),\n Output(display_name=\"Raw Search\", name=\"raw_search\", method=\"raw_search\"),\n ]\n\n def raw_search(self, query: str | dict | None = None) -> Data:\n \"\"\"Execute a raw OpenSearch query against the target index.\n\n Args:\n query (dict[str, Any]): The OpenSearch query DSL dictionary.\n\n Returns:\n Data: Search results as a Data object.\n\n Raises:\n ValueError: If 'query' is not a valid OpenSearch query (must be a non-empty dict).\n \"\"\"\n raw_query = query if query is not None else self.search_query\n\n if raw_query is None or (isinstance(raw_query, str) and not raw_query.strip()):\n self.log(\"No query provided for raw search - returning empty results\")\n return Data(data={})\n\n if isinstance(raw_query, dict):\n query_body = copy.deepcopy(raw_query)\n elif isinstance(raw_query, str):\n s = raw_query.strip()\n\n # First, optimistically try to parse as JSON DSL\n try:\n query_body = json.loads(s)\n except json.JSONDecodeError:\n # Fallback: treat as a basic text query over common fields\n query_body = {\n \"query\": {\n \"multi_match\": {\n \"query\": s,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n }\n }\n }\n else:\n msg = f\"Unsupported raw_search query type: {type(raw_query)!r}\"\n raise TypeError(msg)\n\n filter_obj = self._parse_filter_expression()\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n if filter_clauses:\n if \"query\" in query_body:\n original_query = query_body[\"query\"]\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [original_query],\n \"filter\": filter_clauses,\n }\n }\n else:\n query_body[\"query\"] = {\n \"bool\": {\n \"must\": [{\"match_all\": {}}],\n \"filter\": filter_clauses,\n }\n }\n\n if filter_obj:\n # Apply limit if not already set in the raw query\n if \"size\" not in query_body:\n limit = self._resolve_limit(filter_obj, default_limit=None)\n if limit is not None:\n query_body[\"size\"] = limit\n\n # Apply score_threshold / scoreThreshold as min_score if not already set\n if \"min_score\" not in query_body:\n\n score_threshold = self._resolve_score_threshold(filter_obj)\n if score_threshold is not None:\n\n query_body[\"min_score\"] = score_threshold\n\n client = self.build_client()\n logger.info(f\"query: {query_body}\")\n resp = client.search(\n index=self.index_name,\n body=query_body,\n params={\"terminate_after\": 0},\n )\n # Remove any _source keys whose value is a list of floats (embedding vectors)\n # Minimum length threshold to identify embedding vectors\n min_vector_length = 100\n\n def is_vector(val):\n # Accepts if it's a list of numbers (float or int) and has reasonable vector length\n return (\n isinstance(val, list)\n and len(val) > min_vector_length\n and all(isinstance(x, (float, int)) for x in val)\n )\n\n if \"hits\" in resp and \"hits\" in resp[\"hits\"]:\n for hit in resp[\"hits\"][\"hits\"]:\n source = hit.get(\"_source\")\n if isinstance(source, dict):\n keys_to_remove = [k for k, v in source.items() if is_vector(v)]\n for k in keys_to_remove:\n source.pop(k)\n logger.info(f\"Raw search response (all embedding vectors removed): {resp}\")\n return Data(**resp)\n\n def _get_embedding_model_name(self, embedding_obj=None) -> str:\n \"\"\"Get the embedding model name from component config or embedding object.\n\n Priority: deployment > model > model_id > model_name\n This ensures we use the actual model being deployed, not just the configured model.\n Supports multiple embedding providers (OpenAI, Watsonx, Cohere, etc.)\n\n Args:\n embedding_obj: Specific embedding object to get name from (optional)\n\n Returns:\n Embedding model name\n\n Raises:\n ValueError: If embedding model name cannot be determined\n \"\"\"\n # First try explicit embedding_model_name input\n if hasattr(self, \"embedding_model_name\") and self.embedding_model_name:\n return self.embedding_model_name.strip()\n\n # Try to get from provided embedding object\n if embedding_obj:\n # Priority: deployment > model > model_id > model_name\n if hasattr(embedding_obj, \"deployment\") and embedding_obj.deployment:\n return str(embedding_obj.deployment)\n if hasattr(embedding_obj, \"model\") and embedding_obj.model:\n return str(embedding_obj.model)\n if hasattr(embedding_obj, \"model_id\") and embedding_obj.model_id:\n return str(embedding_obj.model_id)\n if hasattr(embedding_obj, \"model_name\") and embedding_obj.model_name:\n return str(embedding_obj.model_name)\n\n # Try to get from embedding component (legacy single embedding)\n if hasattr(self, \"embedding\") and self.embedding:\n # Handle list of embeddings\n if isinstance(self.embedding, list) and len(self.embedding) > 0:\n first_emb = self.embedding[0]\n if hasattr(first_emb, \"deployment\") and first_emb.deployment:\n return str(first_emb.deployment)\n if hasattr(first_emb, \"model\") and first_emb.model:\n return str(first_emb.model)\n if hasattr(first_emb, \"model_id\") and first_emb.model_id:\n return str(first_emb.model_id)\n if hasattr(first_emb, \"model_name\") and first_emb.model_name:\n return str(first_emb.model_name)\n # Handle single embedding\n elif not isinstance(self.embedding, list):\n if hasattr(self.embedding, \"deployment\") and self.embedding.deployment:\n return str(self.embedding.deployment)\n if hasattr(self.embedding, \"model\") and self.embedding.model:\n return str(self.embedding.model)\n if hasattr(self.embedding, \"model_id\") and self.embedding.model_id:\n return str(self.embedding.model_id)\n if hasattr(self.embedding, \"model_name\") and self.embedding.model_name:\n return str(self.embedding.model_name)\n\n msg = (\n \"Could not determine embedding model name. \"\n \"Please set the 'embedding_model_name' field or ensure the embedding component \"\n \"has a 'deployment', 'model', 'model_id', or 'model_name' attribute.\"\n )\n raise ValueError(msg)\n\n # ---------- helper functions for index management ----------\n def _default_text_mapping(\n self,\n dim: int,\n engine: str = \"jvector\",\n space_type: str = \"l2\",\n ef_search: int = 512,\n ef_construction: int = 100,\n m: int = 16,\n vector_field: str = \"vector_field\",\n ) -> dict[str, Any]:\n \"\"\"Create the default OpenSearch index mapping for vector search.\n\n This method generates the index configuration with k-NN settings optimized\n for approximate nearest neighbor search using the specified vector engine.\n Includes the embedding_model keyword field for tracking which model was used.\n\n Args:\n dim: Dimensionality of the vector embeddings\n engine: Vector search engine (jvector, nmslib, faiss, lucene)\n space_type: Distance metric for similarity calculation\n ef_search: Size of dynamic list used during search\n ef_construction: Size of dynamic list used during index construction\n m: Number of bidirectional links for each vector\n vector_field: Name of the field storing vector embeddings\n\n Returns:\n Dictionary containing OpenSearch index mapping configuration\n \"\"\"\n return {\n \"settings\": {\"index\": {\"knn\": True, \"knn.algo_param.ef_search\": ef_search}},\n \"mappings\": {\n \"properties\": {\n vector_field: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n \"embedding_model\": {\"type\": \"keyword\"}, # Track which model was used\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n },\n }\n\n def _ensure_embedding_field_mapping(\n self,\n client: OpenSearch,\n index_name: str,\n field_name: str,\n dim: int,\n engine: str,\n space_type: str,\n ef_construction: int,\n m: int,\n ) -> None:\n \"\"\"Lazily add a dynamic embedding field to the index if it doesn't exist.\n\n This allows adding new embedding models without recreating the entire index.\n Also ensures the embedding_model tracking field exists.\n\n Note: Some OpenSearch versions/configurations have issues with dynamically adding\n knn_vector mappings (NullPointerException). This method checks if the field\n already exists before attempting to add it, and gracefully skips if the field\n is already properly configured.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index name\n field_name: Dynamic field name for this embedding model\n dim: Vector dimensionality\n engine: Vector search engine\n space_type: Distance metric\n ef_construction: Construction parameter\n m: HNSW parameter\n \"\"\"\n # First, check if the field already exists and is properly mapped\n properties = self._get_index_properties(client)\n if self._is_knn_vector_field(properties, field_name):\n # Field already exists as knn_vector - verify dimensions match\n existing_dim = self._get_field_dimension(properties, field_name)\n if existing_dim is not None and existing_dim != dim:\n logger.warning(\n f\"Field '{field_name}' exists with dimension {existing_dim}, \"\n f\"but current embedding has dimension {dim}. Using existing mapping.\"\n )\n else:\n logger.info(\n f\"[OpenSearchMultimodel] Field '{field_name}' already exists\"\n f\"as knn_vector with matching dimensions - skipping mapping update\"\n )\n return\n\n # Field doesn't exist, try to add the mapping\n try:\n mapping = {\n \"properties\": {\n field_name: {\n \"type\": \"knn_vector\",\n \"dimension\": dim,\n \"method\": {\n \"name\": \"disk_ann\",\n \"space_type\": space_type,\n \"engine\": engine,\n \"parameters\": {\"ef_construction\": ef_construction, \"m\": m},\n },\n },\n # Also ensure the embedding_model tracking field exists as keyword\n \"embedding_model\": {\"type\": \"keyword\"},\n \"embedding_dimensions\": {\"type\": \"integer\"},\n }\n }\n client.indices.put_mapping(index=index_name, body=mapping)\n logger.info(f\"Added/updated embedding field mapping: {field_name}\")\n except RequestError as e:\n error_str = str(e).lower()\n if \"invalid engine\" in error_str and \"jvector\" in error_str:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to OpenSearch 2.9+.\"\n )\n raise ValueError(msg) from e\n if \"index.knn\" in error_str:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from e\n raise\n except Exception as e:\n # Check if this is the known OpenSearch k-NN NullPointerException issue\n error_str = str(e).lower()\n if \"null\" in error_str or \"nullpointerexception\" in error_str:\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}\"\n f\"due to OpenSearch k-NN plugin issue: {e}. \"\n f\"This is a known issue with some OpenSearch versions. \"\n f\"[OpenSearchMultimodel] Skipping mapping update. \"\n f\"Please ensure the index has the correct mapping for KNN search to work.\"\n )\n # Skip and continue - ingestion will proceed, but KNN search may fail if mapping doesn't exist\n return\n logger.warning(\n f\"[OpenSearchMultimodel] Could not add embedding field mapping for {field_name}: {e}\"\n )\n raise\n\n # Verify the field was added correctly\n properties = self._get_index_properties(client)\n if not self._is_knn_vector_field(properties, field_name):\n msg = f\"Field '{field_name}' is not mapped as knn_vector. Current mapping: {properties.get(field_name)}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def _validate_aoss_with_engines(self, *, is_aoss: bool, engine: str) -> None:\n \"\"\"Validate engine compatibility with Amazon OpenSearch Serverless (AOSS).\n\n Amazon OpenSearch Serverless has restrictions on which vector engines\n can be used. This method ensures the selected engine is compatible.\n\n Args:\n is_aoss: Whether the connection is to Amazon OpenSearch Serverless\n engine: The selected vector search engine\n\n Raises:\n ValueError: If AOSS is used with an incompatible engine\n \"\"\"\n if is_aoss and engine not in {\"nmslib\", \"faiss\"}:\n msg = \"Amazon OpenSearch Service Serverless only supports `nmslib` or `faiss` engines\"\n raise ValueError(msg)\n\n def _is_aoss_enabled(self, http_auth: Any) -> bool:\n \"\"\"Determine if Amazon OpenSearch Serverless (AOSS) is being used.\n\n Args:\n http_auth: The HTTP authentication object\n\n Returns:\n True if AOSS is enabled, False otherwise\n \"\"\"\n return (\n http_auth is not None and hasattr(http_auth, \"service\") and http_auth.service == \"aoss\"\n )\n\n @staticmethod\n def _openrag_input_to_str(value: Any) -> str:\n if value is None:\n return \"\"\n if hasattr(value, \"get_secret_value\"):\n value = value.get_secret_value()\n if hasattr(value, \"text\"):\n value = value.text\n return str(value or \"\").strip()\n\n def _openrag_callback_value(self, attr_name: str) -> str:\n value = self._openrag_input_to_str(getattr(self, attr_name, \"\"))\n if value == self._openrag_ingest_global_placeholders.get(attr_name):\n return \"\"\n return value\n\n def _openrag_ingest_callback_config(self) -> tuple[str, str, str] | None:\n url = self._openrag_callback_value(\"openrag_ingest_url\")\n token = self._openrag_callback_value(\"openrag_ingest_token\")\n ingest_run_id = self._openrag_callback_value(\"openrag_ingest_run_id\")\n\n masked_token = (\n f\"{token[:4]}...{token[-4:]}\" if len(token) >= 8 else (\"<set>\" if token else \"\")\n )\n debug_payload = {\n \"openrag_ingest_url\": url,\n \"openrag_ingest_url_len\": len(url),\n \"openrag_ingest_token_masked\": masked_token,\n \"openrag_ingest_token_len\": len(token),\n \"openrag_ingest_run_id\": ingest_run_id,\n \"raw_url_type\": type(self.openrag_ingest_url).__name__,\n \"raw_token_type\": type(self.openrag_ingest_token).__name__,\n \"raw_run_id_type\": type(self.openrag_ingest_run_id).__name__,\n }\n logger.warning(f\"[OpenRAG callback config] {debug_payload}\")\n try:\n self.log(f\"[OpenRAG callback config] {debug_payload}\")\n except Exception:\n pass\n\n if not url and not token and not ingest_run_id:\n return None\n if not url or not token or not ingest_run_id:\n msg = \"OpenRAG ingest callback requires url, token, and ingest_run_id.\"\n raise ValueError(msg)\n return url, token, ingest_run_id\n\n def _post_openrag_ingest_batches(\n self,\n *,\n requests: list[dict],\n vector_field: str,\n text_field: str,\n ) -> None:\n callback_config = self._openrag_ingest_callback_config()\n if callback_config is None:\n return\n\n url, token, ingest_run_id = callback_config\n batch_size = max(self._parse_int_param(\"openrag_ingest_batch_size\", 100), 1)\n timeout = self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT)\n headers = {\"Authorization\": f\"Bearer {token}\"}\n\n masked_token = (\n f\"{token[:4]}...{token[-4:]}\" if len(token) >= 8 else (\"<set>\" if token else \"\")\n )\n request_summary = {\n \"url\": url,\n \"ingest_run_id\": ingest_run_id,\n \"token_masked\": masked_token,\n \"total_chunks\": len(requests),\n \"batch_size\": batch_size,\n \"timeout_s\": timeout,\n }\n logger.warning(f\"[OpenRAG ingest POST] {request_summary}\")\n try:\n self.log(f\"[OpenRAG ingest POST] {request_summary}\")\n except Exception:\n pass\n\n with httpx.Client(timeout=timeout) as client:\n total_batches = (len(requests) + batch_size - 1) // batch_size\n for batch_number, start in enumerate(range(0, len(requests), batch_size), start=1):\n batch = requests[start : start + batch_size]\n final = batch_number == total_batches\n payload = {\n \"ingest_run_id\": ingest_run_id,\n \"batch_id\": batch_number,\n \"final\": final,\n \"chunks\": [\n self._openrag_chunk_payload(\n request,\n vector_field=vector_field,\n text_field=text_field,\n )\n for request in batch\n ],\n }\n logger.warning(\n f\"[OpenRAG ingest POST] -> batch={batch_number}/{total_batches} \"\n f\"url={url} chunks={len(payload['chunks'])} final={final}\"\n )\n response = client.post(url, json=payload, headers=headers)\n response_summary = {\n \"batch\": batch_number,\n \"url\": url,\n \"status\": response.status_code,\n \"final_url\": str(response.request.url),\n \"response_headers\": dict(response.headers),\n \"body_preview\": response.text[:500],\n }\n logger.warning(f\"[OpenRAG ingest POST resp] {response_summary}\")\n try:\n self.log(f\"[OpenRAG ingest POST resp] {response_summary}\")\n except Exception:\n pass\n if response.status_code >= 400:\n msg = (\n \"OpenRAG ingest callback failed \"\n f\"(batch={batch_number}, status={response.status_code}, \"\n f\"url={url}): {response.text[:1000]}\"\n )\n raise RuntimeError(msg)\n\n self.log(f\"Posted {len(requests)} chunks to OpenRAG backend ingest callback.\")\n\n @staticmethod\n def _openrag_chunk_payload(\n request: dict,\n *,\n vector_field: str,\n text_field: str,\n ) -> dict:\n metadata = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\", vector_field, text_field}\n }\n page = metadata.get(\"page\")\n if isinstance(page, str) and page.isdigit():\n page = int(page)\n return {\n \"id\": request.get(\"_id\") or request.get(\"id\"),\n \"text\": request.get(text_field, \"\"),\n \"vector\": request[vector_field],\n \"page\": page if isinstance(page, int) else None,\n \"metadata\": metadata,\n }\n\n def _bulk_ingest_embeddings(\n self,\n client: OpenSearch,\n index_name: str,\n embeddings: list[list[float]],\n texts: list[str],\n metadatas: list[dict] | None = None,\n ids: list[str] | None = None,\n vector_field: str = \"vector_field\",\n text_field: str = \"text\",\n embedding_model: str = \"unknown\",\n mapping: dict | None = None,\n max_chunk_bytes: int | None = 1 * 1024 * 1024,\n *,\n is_aoss: bool = False,\n ) -> list[str]:\n \"\"\"Efficiently ingest multiple documents with embeddings into OpenSearch.\n\n This method uses bulk operations to insert documents with their vector\n embeddings and metadata into the specified OpenSearch index. Each document\n is tagged with the embedding_model name for tracking.\n\n Args:\n client: OpenSearch client instance\n index_name: Target index for document storage\n embeddings: List of vector embeddings for each document\n texts: List of document texts\n metadatas: Optional metadata dictionaries for each document\n ids: Optional document IDs (UUIDs generated if not provided)\n vector_field: Field name for storing vector embeddings\n text_field: Field name for storing document text\n embedding_model: Name of the embedding model used\n mapping: Optional index mapping configuration\n max_chunk_bytes: Maximum size per bulk request chunk\n is_aoss: Whether using Amazon OpenSearch Serverless\n\n Returns:\n List of document IDs that were successfully ingested\n \"\"\"\n logger.debug(f\"[OpenSearchMultimodel] Bulk ingesting embeddings for {index_name}\")\n if not mapping:\n mapping = {}\n\n requests = []\n return_ids = []\n vector_dimensions = len(embeddings[0]) if embeddings else None\n\n for i, text in enumerate(texts):\n metadata = metadatas[i] if metadatas else {}\n if vector_dimensions is not None and \"embedding_dimensions\" not in metadata:\n metadata = {**metadata, \"embedding_dimensions\": vector_dimensions}\n\n # Normalize ACL fields that may arrive as JSON strings from flows\n for key in (\"allowed_users\", \"allowed_groups\", \"allowed_principals\"):\n value = metadata.get(key)\n if isinstance(value, str):\n try:\n parsed = json.loads(value)\n if isinstance(parsed, list):\n metadata[key] = parsed\n except (json.JSONDecodeError, TypeError):\n # Leave value as-is if it isn't valid JSON\n pass\n\n metadata_document_id = str(metadata.get(\"document_id\") or \"\").strip()\n if metadata_document_id and metadata_document_id.lower() != \"none\":\n generated_id = f\"{metadata_document_id}_{i}\"\n else:\n generated_id = str(uuid.uuid4())\n _id = ids[i] if ids else generated_id\n request = {\n \"_op_type\": \"index\",\n \"_index\": index_name,\n vector_field: embeddings[i],\n text_field: text,\n \"embedding_model\": embedding_model, # Track which model was used\n **metadata,\n }\n if is_aoss:\n request[\"id\"] = _id\n else:\n request[\"_id\"] = _id\n requests.append(request)\n return_ids.append(_id)\n if metadatas:\n self.log(f\"Sample metadata: {metadatas[0] if metadatas else {}}\")\n if self._openrag_ingest_callback_config() is not None:\n self._post_openrag_ingest_batches(\n requests=requests,\n vector_field=vector_field,\n text_field=text_field,\n )\n return return_ids\n try:\n helpers.bulk(client, requests, max_chunk_bytes=max_chunk_bytes)\n except Exception as bulk_error:\n if \"Unsupported request type for filter level DLS\" not in str(bulk_error):\n raise\n logger.warning(\n \"[OpenSearchMultimodel] Bulk ingest is blocked by filter-level DLS; \"\n \"falling back to per-document index requests.\"\n )\n self._index_embeddings_individually(client, requests)\n return return_ids\n\n def _index_embeddings_individually(\n self,\n client: OpenSearch,\n requests: list[dict],\n ) -> None:\n \"\"\"Index documents one at a time when OpenSearch DLS rejects bulk writes.\"\"\"\n for request in requests:\n document_id = request.get(\"_id\") or request.get(\"id\")\n body = {\n key: value\n for key, value in request.items()\n if key not in {\"_op_type\", \"_index\", \"_id\", \"id\"}\n }\n client.index(index=request[\"_index\"], id=document_id, body=body)\n\n def _log_index_admin_skip(self, operation: str, error: Exception) -> None:\n \"\"\"Log index-admin operations that may be blocked under filter-level DLS.\"\"\"\n logger.warning(\n f\"[OpenSearchMultimodel] Could not run index-admin operation '{operation}': {error}. \"\n \"Assuming the backend pre-created the required index/mapping and continuing.\"\n )\n\n # ---------- param helpers ----------\n def _parse_int_param(self, attr_name: str, default: int) -> int:\n \"\"\"Parse a string attribute to int, returning *default* on failure.\"\"\"\n raw = getattr(self, attr_name, None)\n if raw is None or str(raw).strip() == \"\":\n return default\n try:\n value = int(str(raw).strip())\n except ValueError:\n logger.warning(\n f\"Invalid integer value '{raw}' for {attr_name}, using default {default}\"\n )\n return default\n\n if value < 0:\n logger.warning(f\"Negative value '{raw}' for {attr_name}, using default {default}\")\n return default\n\n return value\n\n # ---------- auth / client ----------\n def _build_auth_kwargs(self) -> dict[str, Any]:\n \"\"\"Build authentication configuration for OpenSearch client.\n\n Constructs the appropriate authentication parameters based on the\n selected auth mode (basic username/password or JWT token).\n\n Returns:\n Dictionary containing authentication configuration\n\n Raises:\n ValueError: If required authentication parameters are missing\n \"\"\"\n mode = (self.auth_mode or \"basic\").strip().lower()\n if mode == \"jwt\":\n token = (self.jwt_token or \"\").strip()\n if not token:\n msg = \"Auth Mode is 'jwt' but no jwt_token was provided.\"\n raise ValueError(msg)\n header_name = (self.jwt_header or \"Authorization\").strip()\n header_value = f\"Bearer {token}\" if self.bearer_prefix else token\n return {\"headers\": {header_name: header_value}}\n if mode == \"openrag\":\n # Writes are delegated to the OpenRAG backend ingest callback,\n # so no direct OpenSearch credentials are needed. Only the\n # OPENRAG_* fields are required for ingestion to function.\n missing = [\n name\n for name, value in (\n (\"openrag_ingest_url\", self.openrag_ingest_url),\n (\"openrag_ingest_token\", self.openrag_ingest_token),\n (\"openrag_ingest_run_id\", self.openrag_ingest_run_id),\n )\n if not (value or \"\").strip()\n ]\n if missing:\n msg = (\n \"Auth Mode is 'openrag' but required OPENRAG_* fields are \"\n f\"missing: {', '.join(missing)}.\"\n )\n raise ValueError(msg)\n return {}\n user = (self.username or \"\").strip()\n pwd = (self.password or \"\").strip()\n if not user or not pwd:\n msg = \"Auth Mode is 'basic' but username/password are missing.\"\n raise ValueError(msg)\n return {\"http_auth\": (user, pwd)}\n\n def build_client(self) -> OpenSearch:\n \"\"\"Create and configure an OpenSearch client instance.\n\n Returns:\n Configured OpenSearch client ready for operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel] Building OpenSearch client\")\n auth_kwargs = self._build_auth_kwargs()\n return OpenSearch(\n hosts=[self.opensearch_url],\n use_ssl=self.use_ssl,\n verify_certs=self.verify_certs,\n ssl_assert_hostname=False,\n ssl_show_warn=False,\n timeout=self._parse_int_param(\"request_timeout\", REQUEST_TIMEOUT),\n max_retries=self._parse_int_param(\"max_retries\", MAX_RETRIES),\n retry_on_timeout=True,\n **auth_kwargs,\n )\n\n @check_cached_vector_store\n def build_vector_store(self) -> OpenSearch:\n # Return raw OpenSearch client as our \"vector store.\"\n client = self.build_client()\n\n # Check if we're in ingestion-only mode (no search query)\n has_search_query = bool((self.search_query or \"\").strip())\n if not has_search_query:\n logger.debug(\n \"[OpenSearchMultimodel] Ingestion-only mode activated: search operations will be skipped\"\n )\n logger.debug(\"[OpenSearchMultimodel] Starting ingestion mode...\")\n\n logger.debug(f\"[OpenSearchMultimodel] Embedding: {self.embedding}\")\n self._add_documents_to_vector_store(client=client)\n return client\n\n # ---------- ingest ----------\n def _add_documents_to_vector_store(self, client: OpenSearch) -> None:\n \"\"\"Process and ingest documents into the OpenSearch vector store.\n\n This method handles the complete document ingestion pipeline:\n - Prepares document data and metadata\n - Generates vector embeddings using the selected model\n - Creates appropriate index mappings with dynamic field names\n - Bulk inserts documents with vectors and model tracking\n\n Args:\n client: OpenSearch client for performing operations\n \"\"\"\n logger.debug(\"[OpenSearchMultimodel][INGESTION] _add_documents_to_vector_store called\")\n # Convert DataFrame to Data if needed using parent's method\n self.ingest_data = self._prepare_ingest_data()\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data type: \"\n f\"{type(self.ingest_data)}, length: {len(self.ingest_data) if self.ingest_data else 0}\"\n )\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] ingest_data content: \"\n f\"{self.ingest_data[:2] if self.ingest_data and len(self.ingest_data) > 0 else 'empty'}\"\n )\n\n docs = self.ingest_data or []\n if not docs:\n logger.debug(\"Ingestion complete: No documents provided\")\n return\n\n if not self.embedding:\n msg = \"Embedding handle is required to embed documents.\"\n raise ValueError(msg)\n\n # Normalize embedding to list first\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n\n # Filter out None values (fail-safe mode) - do this BEFORE checking if empty\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n # NOW check if we have any valid embeddings left after filtering\n if not embeddings_list:\n logger.warning(\n \"All embeddings returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n self.log(\n \"Embedding returned None (fail-safe mode enabled). Skipping document ingestion.\"\n )\n return\n\n logger.debug(\n f\"[OpenSearchMultimodel][INGESTION] Valid embeddings after filtering: {len(embeddings_list)}\"\n )\n self.log(\n f\"[OpenSearchMultimodel][INGESTION] Available embedding models: {len(embeddings_list)}\"\n )\n\n # Select the embedding to use for ingestion\n selected_embedding = None\n embedding_model = None\n\n # If embedding_model_name is specified, find matching embedding\n if (\n hasattr(self, \"embedding_model_name\")\n and self.embedding_model_name\n and self.embedding_model_name.strip()\n ):\n target_model_name = self.embedding_model_name.strip()\n self.log(f\"Looking for embedding model: {target_model_name}\")\n\n for emb_obj in embeddings_list:\n # Check all possible model identifiers (deployment, model, model_id, model_name)\n # Also check available_models list from EmbeddingsWithModels\n possible_names = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n if deployment:\n possible_names.append(str(deployment))\n if model:\n possible_names.append(str(model))\n if model_id:\n possible_names.append(str(model_id))\n if model_name:\n possible_names.append(str(model_name))\n\n # Also add combined identifier\n if deployment and model and deployment != model:\n possible_names.append(f\"{deployment}:{model}\")\n\n # Add all models from available_models dict\n if available_models_attr and isinstance(available_models_attr, dict):\n possible_names.extend(\n str(model_key).strip()\n for model_key in available_models_attr\n if model_key and str(model_key).strip()\n )\n\n # Match if target matches any of the possible names\n if target_model_name in possible_names:\n # Check if target is in available_models dict - use dedicated instance\n if (\n available_models_attr\n and isinstance(available_models_attr, dict)\n and target_model_name in available_models_attr\n ):\n # Use the dedicated embedding instance from the dict\n selected_embedding = available_models_attr[target_model_name]\n embedding_model = target_model_name\n self.log(\n f\"Found dedicated embedding instance for '{embedding_model}' in available_models dict\"\n )\n else:\n # Traditional identifier match\n selected_embedding = emb_obj\n embedding_model = self._get_embedding_model_name(emb_obj)\n self.log(\n f\"Found matching embedding model: {embedding_model} (matched on: {target_model_name})\"\n )\n break\n\n if not selected_embedding:\n # Build detailed list of available embeddings with all their identifiers\n available_info = []\n for idx, emb in enumerate(embeddings_list):\n emb_type = type(emb).__name__\n identifiers = []\n deployment = getattr(emb, \"deployment\", None)\n model = getattr(emb, \"model\", None)\n model_id = getattr(emb, \"model_id\", None)\n model_name = getattr(emb, \"model_name\", None)\n available_models_attr = getattr(emb, \"available_models\", None)\n\n if deployment:\n identifiers.append(f\"deployment='{deployment}'\")\n if model:\n identifiers.append(f\"model='{model}'\")\n if model_id:\n identifiers.append(f\"model_id='{model_id}'\")\n if model_name:\n identifiers.append(f\"model_name='{model_name}'\")\n\n # Add combined identifier as an option\n if deployment and model and deployment != model:\n identifiers.append(f\"combined='{deployment}:{model}'\")\n\n # Add available_models dict if present\n if available_models_attr and isinstance(available_models_attr, dict):\n identifiers.append(f\"available_models={list(available_models_attr.keys())}\")\n\n available_info.append(\n f\" [{idx}] {emb_type}: {', '.join(identifiers) if identifiers else 'No identifiers'}\"\n )\n\n msg = (\n f\"Embedding model '{target_model_name}' not found in available embeddings.\\n\\n\"\n f\"Available embeddings:\\n\" + \"\\n\".join(available_info) + \"\\n\\n\"\n \"Please set 'embedding_model_name' to one of the identifier values shown above \"\n \"(use the value after the '=' sign, without quotes).\\n\"\n \"For duplicate deployments, use the 'combined' format.\\n\"\n \"Or leave it empty to use the first embedding.\"\n )\n raise ValueError(msg)\n else:\n # Use first embedding if no model name specified\n selected_embedding = embeddings_list[0]\n embedding_model = self._get_embedding_model_name(selected_embedding)\n self.log(f\"No embedding_model_name specified, using first embedding: {embedding_model}\")\n\n dynamic_field_name = get_embedding_field_name(embedding_model)\n\n logger.info(f\"Selected embedding model for ingestion: '{embedding_model}'\")\n self.log(f\"Using embedding model for ingestion: {embedding_model}\")\n self.log(f\"Dynamic vector field: {dynamic_field_name}\")\n\n # Log embedding details for debugging\n if hasattr(selected_embedding, \"deployment\"):\n logger.info(f\"Embedding deployment: {selected_embedding.deployment}\")\n if hasattr(selected_embedding, \"model\"):\n logger.info(f\"Embedding model: {selected_embedding.model}\")\n if hasattr(selected_embedding, \"model_id\"):\n logger.info(f\"Embedding model_id: {selected_embedding.model_id}\")\n if hasattr(selected_embedding, \"dimensions\"):\n logger.info(f\"Embedding dimensions: {selected_embedding.dimensions}\")\n if hasattr(selected_embedding, \"available_models\"):\n logger.info(f\"Embedding available_models: {selected_embedding.available_models}\")\n\n # No model switching needed - each model in available_models has its own dedicated instance\n # The selected_embedding is already configured correctly for the target model\n logger.info(\n f\"Using embedding instance for '{embedding_model}' - pre-configured and ready to use\"\n )\n\n # Extract texts and metadata from documents\n texts = []\n metadatas = []\n # Process docs_metadata table input into a dict\n additional_metadata = {}\n logger.debug(f\"[LF] Docs metadata {self.docs_metadata}\")\n if hasattr(self, \"docs_metadata\") and self.docs_metadata:\n logger.info(f\"[LF] Docs metadata {self.docs_metadata}\")\n if isinstance(self.docs_metadata[-1], Data):\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n self.docs_metadata = self.docs_metadata[-1].data\n logger.info(f\"[LF] Docs metadata is a Data object {self.docs_metadata}\")\n additional_metadata.update(self.docs_metadata)\n else:\n for item in self.docs_metadata:\n if isinstance(item, dict) and \"key\" in item and \"value\" in item:\n additional_metadata[item[\"key\"]] = item[\"value\"]\n # Replace string \"None\" values with actual None\n for key, value in additional_metadata.items():\n if value == \"None\":\n additional_metadata[key] = None\n logger.info(f\"[LF] Additional metadata {additional_metadata}\")\n for doc_obj in docs:\n data_copy = json.loads(doc_obj.model_dump_json())\n text = data_copy.pop(doc_obj.text_key, doc_obj.default_value)\n texts.append(text)\n\n # Merge additional metadata from table input\n data_copy.update(additional_metadata)\n\n metadatas.append(data_copy)\n self.log(metadatas)\n\n # Determine whether the selected embedding is watsonx/IBM. The watsonx\n # SDK ships its own rate-limit machinery (input batching, proactive\n # x-requests-limit-* TokenBucket throttling, and jittered exponential\n # backoff on 429), so we lean on it instead of retrying on top of it.\n # The type-name check also covers watsonx-hosted, non-\"ibm/\" models\n # (e.g. intfloat/multilingual-e5-large).\n is_ibm = (embedding_model and \"ibm\" in str(embedding_model).lower()) or (\n selected_embedding and \"watsonx\" in type(selected_embedding).__name__.lower()\n )\n logger.debug(f\"Is IBM/watsonx embedding: {is_ibm}\")\n\n if is_ibm:\n\n # Hand the full batch to the SDK and let it batch/throttle/retry.\n # Retry attempts and base backoff are tunable via the SDK's own\n # WATSONX_MAX_RETRIES / WATSONX_DELAY_TIME environment variables.\n logger.info(\n f\"Embedding {len(texts)} chunks via watsonx SDK batch (SDK-managed throttle + 429 retry)\"\n )\n try:\n vectors: list[list[float]] = selected_embedding.embed_documents(texts)\n logger.info(f\"Successfully embedded {len(vectors)} chunks via watsonx SDK\")\n except Exception as embed_error:\n _log_watsonx_rate_limit_headers(embed_error)\n logger.error(\n f\"Failed to embed {len(texts)} chunks via watsonx SDK. Error: {embed_error}\",\n )\n raise\n\n else:\n # Non-watsonx providers (OpenAI, Ollama) lack the watsonx SDK's\n # built-in rate-limit handling, so embed per chunk in parallel with\n # a generic rate-limit-aware tenacity retry.\n vectors: list[list[float]] = [None] * len(texts)\n from tenacity import (\n retry,\n retry_if_exception,\n stop_after_attempt,\n wait_exponential,\n )\n\n def is_rate_limit_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a rate limit error (429).\"\"\"\n error_str = str(exception).lower()\n return \"429\" in error_str or \"rate_limit\" in error_str or \"rate limit\" in error_str\n\n def is_other_retryable_error(exception: Exception) -> bool:\n \"\"\"Check if exception is a transient network error worth retrying.\"\"\"\n if is_rate_limit_error(exception):\n return False\n return isinstance(exception, (ConnectionError, TimeoutError, OSError))\n\n # Retry decorator for rate limit errors (longer backoff)\n retry_on_rate_limit = retry(\n retry=retry_if_exception(is_rate_limit_error),\n stop=stop_after_attempt(5),\n wait=wait_exponential(multiplier=2, min=2, max=30),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Rate limit hit for chunk (attempt {retry_state.attempt_number}/5), \"\n f\"backing off for {retry_state.next_action.sleep:.1f}s\"\n ),\n )\n\n # Retry decorator for other errors (shorter backoff)\n retry_on_other_errors = retry(\n retry=retry_if_exception(is_other_retryable_error),\n stop=stop_after_attempt(3),\n wait=wait_exponential(multiplier=1, min=1, max=8),\n reraise=True,\n before_sleep=lambda retry_state: logger.warning(\n f\"Error embedding chunk (attempt {retry_state.attempt_number}/3), \"\n f\"retrying in {retry_state.next_action.sleep:.1f}s: {retry_state.outcome.exception()}\"\n ),\n )\n\n def embed_chunk_with_retry(chunk_text: str, chunk_idx: int) -> list[float]:\n \"\"\"Embed a single chunk with rate-limit-aware retry logic.\"\"\"\n\n @retry_on_rate_limit\n @retry_on_other_errors\n def _embed(text: str) -> list[float]:\n return selected_embedding.embed_documents([text])[0]\n\n try:\n return _embed(chunk_text)\n except Exception as e:\n logger.error(\n f\"Failed to embed chunk {chunk_idx} after all retries: {e}\",\n error=str(e),\n )\n raise\n\n max_workers = min(max(len(texts), 1), 8)\n logger.debug(f\"Using parallel processing with {max_workers} workers\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n futures = {\n executor.submit(embed_chunk_with_retry, chunk, idx): idx\n for idx, chunk in enumerate(texts)\n }\n for future in as_completed(futures):\n idx = futures[future]\n vectors[idx] = future.result()\n\n if not vectors:\n self.log(f\"No vectors generated from documents for model {embedding_model}.\")\n return\n\n # Get vector dimension for mapping\n dim = len(vectors[0]) if vectors else 768 # default fallback\n use_openrag_ingest_callback = self._openrag_ingest_callback_config() is not None\n\n is_aoss = False\n mapping: dict | None = None\n\n engine = getattr(self, \"engine\", \"jvector\")\n\n if use_openrag_ingest_callback:\n self.log(\"Using OpenRAG backend ingest callback; skipping direct OpenSearch writes.\")\n else:\n # Check for AOSS\n auth_kwargs = self._build_auth_kwargs()\n is_aoss = self._is_aoss_enabled(auth_kwargs.get(\"http_auth\"))\n\n # Validate engine with AOSS\n self._validate_aoss_with_engines(is_aoss=is_aoss, engine=engine)\n\n # Create mapping with proper KNN settings\n space_type = getattr(self, \"space_type\", \"l2\")\n ef_construction = getattr(self, \"ef_construction\", 512)\n m = getattr(self, \"m\", 16)\n\n mapping = self._default_text_mapping(\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n vector_field=dynamic_field_name, # Use dynamic field name\n )\n\n # Ensure index exists with baseline mapping (index.knn: true is required for vector search)\n index_exists = True\n try:\n index_exists = bool(client.indices.exists(index=self.index_name))\n except OpenSearchException as exists_error:\n self._log_index_admin_skip(\"indices.exists\", exists_error)\n\n try:\n if not index_exists:\n self.log(f\"Creating index '{self.index_name}' with base mapping\")\n client.indices.create(index=self.index_name, body=mapping)\n except RequestError as creation_error:\n if creation_error.error == \"resource_already_exists_exception\":\n pass # Index was created concurrently\n else:\n error_msg = str(creation_error).lower()\n if \"invalid engine\" in error_msg or \"illegal_argument\" in error_msg:\n if \"jvector\" in error_msg:\n msg = (\n \"The 'jvector' engine is not available in your OpenSearch installation. \"\n \"Use 'nmslib' or 'faiss' for standard OpenSearch, or upgrade to 2.9+.\"\n )\n raise ValueError(msg) from creation_error\n if \"index.knn\" in error_msg:\n msg = (\n \"The index has index.knn: false. Delete the existing index and let the \"\n \"component recreate it, or create a new index with a different name.\"\n )\n raise ValueError(msg) from creation_error\n logger.warning(f\"Failed to create index '{self.index_name}': {creation_error}\")\n raise\n\n # Ensure the dynamic field exists in the index\n self._ensure_embedding_field_mapping(\n client=client,\n index_name=self.index_name,\n field_name=dynamic_field_name,\n dim=dim,\n engine=engine,\n space_type=space_type,\n ef_construction=ef_construction,\n m=m,\n )\n\n self.log(\n f\"Indexing {len(texts)} documents into '{self.index_name}' with model '{embedding_model}'...\"\n )\n logger.info(f\"Will store embeddings in field: {dynamic_field_name}\")\n logger.info(f\"Will tag documents with embedding_model: {embedding_model}\")\n\n # Use the bulk ingestion with model tracking\n return_ids = self._bulk_ingest_embeddings(\n client=client,\n index_name=self.index_name,\n embeddings=vectors,\n texts=texts,\n metadatas=metadatas,\n vector_field=dynamic_field_name, # Use dynamic field name\n text_field=\"text\",\n embedding_model=embedding_model, # Track the model\n mapping=mapping,\n is_aoss=is_aoss,\n )\n self.log(metadatas)\n\n logger.info(\n f\"Ingestion complete: Successfully indexed {len(return_ids)} documents with model '{embedding_model}'\"\n )\n self.log(f\"Successfully indexed {len(return_ids)} documents with model {embedding_model}.\")\n\n # ---------- helpers for filters ----------\n def _is_placeholder_term(self, term_obj: dict) -> bool:\n # term_obj like {\"filename\": \"__IMPOSSIBLE_VALUE__\"}\n return any(v == \"__IMPOSSIBLE_VALUE__\" for v in term_obj.values())\n\n def _coerce_filter_clauses(self, filter_obj: dict | None) -> list[dict]:\n \"\"\"Convert filter expressions into OpenSearch-compatible filter clauses.\n\n This method accepts two filter formats and converts them to standardized\n OpenSearch query clauses:\n\n Format A - Explicit filters:\n {\"filter\": [{\"term\": {\"field\": \"value\"}}, {\"terms\": {\"field\": [\"val1\", \"val2\"]}}],\n \"limit\": 10, \"score_threshold\": 1.5}\n\n Format B - Context-style mapping:\n {\"data_sources\": [\"file1.pdf\"], \"document_types\": [\"pdf\"], \"owners\": [\"user1\"]}\n\n Args:\n filter_obj: Filter configuration dictionary or None\n\n Returns:\n List of OpenSearch filter clauses (term/terms objects)\n Placeholder values with \"__IMPOSSIBLE_VALUE__\" are ignored\n \"\"\"\n if not filter_obj:\n return []\n\n # If it is a string, try to parse it once\n if isinstance(filter_obj, str):\n try:\n filter_obj = json.loads(filter_obj)\n except json.JSONDecodeError:\n # Not valid JSON - treat as no filters\n return []\n\n # Case A: already an explicit list/dict under \"filter\"\n if \"filter\" in filter_obj:\n raw = filter_obj[\"filter\"]\n if isinstance(raw, dict):\n raw = [raw]\n explicit_clauses: list[dict] = []\n for f in raw or []:\n if (\n \"term\" in f\n and isinstance(f[\"term\"], dict)\n and not self._is_placeholder_term(f[\"term\"])\n ):\n explicit_clauses.append(f)\n elif \"terms\" in f and isinstance(f[\"terms\"], dict):\n field, vals = next(iter(f[\"terms\"].items()))\n if isinstance(vals, list) and len(vals) > 0:\n explicit_clauses.append(f)\n return explicit_clauses\n\n # Case B: convert context-style maps into clauses\n field_mapping = {\n \"data_sources\": \"filename\",\n \"document_types\": \"mimetype\",\n \"owners\": \"owner\",\n }\n context_clauses: list[dict] = []\n for k, values in filter_obj.items():\n if not isinstance(values, list):\n continue\n field = field_mapping.get(k, k)\n if len(values) == 0:\n # Match-nothing placeholder (kept to mirror your tool semantics)\n context_clauses.append({\"term\": {field: \"__IMPOSSIBLE_VALUE__\"}})\n elif len(values) == 1:\n if values[0] != \"__IMPOSSIBLE_VALUE__\":\n context_clauses.append({\"term\": {field: values[0]}})\n else:\n context_clauses.append({\"terms\": {field: values}})\n return context_clauses\n\n\n def _parse_filter_expression(self) -> dict | None:\n \"\"\"Parse and validate optional filter_expression JSON.\n\n Returns:\n Parsed JSON object as a dict, or None when unset/blank.\n\n Raises:\n ValueError: If JSON is invalid or does not decode to an object.\n \"\"\"\n filter_expression = getattr(self, \"filter_expression\", \"\")\n if not isinstance(filter_expression, str) or not filter_expression.strip():\n return None\n try:\n filter_obj = json.loads(filter_expression)\n except json.JSONDecodeError as e:\n msg = f\"Invalid filter_expression JSON: {e}\"\n raise ValueError(msg) from e\n\n if not isinstance(filter_obj, dict):\n msg = \"Invalid filter_expression JSON type: expected a JSON object.\"\n raise TypeError(msg)\n return filter_obj\n\n def _resolve_limit(self, filter_obj: dict | None, default_limit: int | None) -> int | None:\n \"\"\"Resolve an integer result limit from filter settings.\"\"\"\n if not filter_obj:\n return default_limit\n raw_limit = filter_obj.get(\"limit\", default_limit)\n if raw_limit is None:\n return None\n if isinstance(raw_limit, bool):\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise TypeError(msg)\n try:\n limit = int(raw_limit)\n except (TypeError, ValueError) as e:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg) from e\n if limit <= 0:\n msg = \"Invalid filter_expression.limit: expected a positive integer.\"\n raise ValueError(msg)\n return limit\n\n def _resolve_score_threshold(self, filter_obj: dict | None) -> float | None:\n \"\"\"Resolve optional positive min score from filter settings.\"\"\"\n if not filter_obj:\n return None\n score_threshold = filter_obj.get(\"score_threshold\")\n if score_threshold is None:\n score_threshold = filter_obj.get(\"scoreThreshold\")\n if not isinstance(score_threshold, (int, float)) or score_threshold <= 0:\n return None\n return float(score_threshold)\n\n def _detect_available_models(self, client: OpenSearch, filter_clauses: list[dict] | None = None) -> list[str]:\n\n \"\"\"Detect which embedding models have documents in the index.\n\n Uses aggregation to find all unique embedding_model values, optionally\n filtered to only documents matching the user's filter criteria.\n\n Args:\n client: OpenSearch client instance\n filter_clauses: Optional filter clauses to scope model detection\n\n Returns:\n List of embedding model names found in the index\n \"\"\"\n try:\n agg_query = {\n \"size\": 0,\n \"aggs\": {\"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}}},\n }\n\n # Apply filters to model detection if any exist\n if filter_clauses:\n agg_query[\"query\"] = {\"bool\": {\"filter\": filter_clauses}}\n\n logger.debug(f\"Model detection query: {agg_query}\")\n result = client.search(\n index=self.index_name,\n body=agg_query,\n params={\"terminate_after\": 0},\n )\n buckets = result.get(\"aggregations\", {}).get(\"embedding_models\", {}).get(\"buckets\", [])\n models = [b[\"key\"] for b in buckets if b[\"key\"]]\n\n # Log detailed bucket info for debugging\n logger.info(\n f\"Detected embedding models in corpus: {models}\"\n + (f\" (with {len(filter_clauses)} filters)\" if filter_clauses else \"\")\n )\n if not models:\n total_hits = result.get(\"hits\", {}).get(\"total\", {})\n total_count = (\n total_hits.get(\"value\", 0) if isinstance(total_hits, dict) else total_hits\n )\n logger.warning(\n f\"No embedding_model values found in index '{self.index_name}'. \"\n f\"Total docs in index: {total_count}. \"\n f\"This may indicate documents were indexed without the embedding_model field.\"\n )\n except (OpenSearchException, KeyError, ValueError) as e:\n logger.warning(f\"Failed to detect embedding models: {e}\")\n # Fallback to current model\n fallback_model = self._get_embedding_model_name()\n logger.info(f\"Using fallback model: {fallback_model}\")\n return [fallback_model]\n else:\n return models\n\n def _get_index_properties(self, client: OpenSearch) -> dict[str, Any] | None:\n \"\"\"Retrieve flattened mapping properties for the current index.\"\"\"\n try:\n mapping = client.indices.get_mapping(index=self.index_name)\n except OpenSearchException as e:\n logger.warning(\n f\"Failed to fetch mapping for index '{self.index_name}': {e}. Proceeding without mapping metadata.\"\n )\n return None\n\n properties: dict[str, Any] = {}\n for index_data in mapping.values():\n props = index_data.get(\"mappings\", {}).get(\"properties\", {})\n if isinstance(props, dict):\n properties.update(props)\n return properties\n\n def _is_knn_vector_field(self, properties: dict[str, Any] | None, field_name: str) -> bool:\n \"\"\"Check whether the field is mapped as a knn_vector.\"\"\"\n if not field_name:\n return False\n if properties is None:\n logger.warning(\n f\"Mapping metadata unavailable; assuming field '{field_name}' is usable.\"\n )\n return True\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return False\n if field_def.get(\"type\") == \"knn_vector\":\n return True\n\n nested_props = field_def.get(\"properties\")\n return bool(isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\")\n\n def _get_field_dimension(\n self, properties: dict[str, Any] | None, field_name: str\n ) -> int | None:\n \"\"\"Get the dimension of a knn_vector field from the index mapping.\n\n Args:\n properties: Index properties from mapping\n field_name: Name of the vector field\n\n Returns:\n Dimension of the field, or None if not found\n \"\"\"\n if not field_name or properties is None:\n return None\n\n field_def = properties.get(field_name)\n if not isinstance(field_def, dict):\n return None\n\n # Check direct knn_vector field\n if field_def.get(\"type\") == \"knn_vector\":\n return field_def.get(\"dimension\")\n\n # Check nested properties\n nested_props = field_def.get(\"properties\")\n if isinstance(nested_props, dict) and nested_props.get(\"type\") == \"knn_vector\":\n return nested_props.get(\"dimension\")\n\n return None\n\n def _get_filename_agg_field(self, index_properties: dict[str, Any] | None) -> str:\n \"\"\"Choose the appropriate field for filename aggregations.\"\"\"\n if not index_properties:\n return \"filename.keyword\"\n\n filename_def = index_properties.get(\"filename\")\n if not isinstance(filename_def, dict):\n return \"filename.keyword\"\n\n field_type = filename_def.get(\"type\")\n fields_def = filename_def.get(\"fields\", {})\n\n # Top-level keyword with no subfields\n if field_type == \"keyword\" and not isinstance(fields_def, dict):\n return \"filename\"\n\n # Text field with keyword subfield\n if isinstance(fields_def, dict) and \"keyword\" in fields_def:\n return \"filename.keyword\"\n\n # Fallback: aggregate on filename directly\n return \"filename\"\n\n # ---------- search (multi-model hybrid) ----------\n def search(self, query: str | None = None) -> list[dict[str, Any]]:\n \"\"\"Perform multi-model hybrid search combining multiple vector similarities and keyword matching.\n\n This method executes a sophisticated search that:\n 1. Auto-detects all embedding models present in the index\n 2. Generates query embeddings for ALL detected models in parallel\n 3. Combines multiple KNN queries using dis_max (picks best match)\n 4. Adds keyword search with fuzzy matching (30% weight)\n 5. Applies optional filtering and score thresholds\n 6. Returns aggregations for faceted search\n\n Search weights:\n - Semantic search (dis_max across all models): 70%\n - Keyword search: 30%\n\n Args:\n query: Search query string (used for both vector embedding and keyword search)\n\n Returns:\n List of search results with page_content, metadata, and relevance scores\n\n Raises:\n ValueError: If embedding component is not provided or filter JSON is invalid\n \"\"\"\n logger.info(self.ingest_data)\n client = self.build_client()\n q = (query or \"\").strip()\n\n # Parse optional filter expression\n filter_obj = self._parse_filter_expression()\n\n if not self.embedding:\n msg = \"Embedding is required to run hybrid search (KNN + keyword).\"\n raise ValueError(msg)\n\n # Check if embedding is None (fail-safe mode)\n if self.embedding is None or (\n isinstance(self.embedding, list) and all(e is None for e in self.embedding)\n ):\n logger.error(\"Embedding returned None (fail-safe mode enabled). Cannot perform search.\")\n return []\n\n # Build filter clauses first so we can use them in model detection\n filter_clauses = self._coerce_filter_clauses(filter_obj)\n\n # Detect available embedding models in the index (scoped by filters)\n available_models = self._detect_available_models(client, filter_clauses)\n\n if not available_models:\n logger.warning(\"No embedding models found in index, using current model\")\n available_models = [self._get_embedding_model_name()]\n\n # Generate embeddings for ALL detected models\n query_embeddings = {}\n\n # Normalize embedding to list\n embeddings_list = self.embedding if isinstance(self.embedding, list) else [self.embedding]\n # Filter out None values (fail-safe mode)\n embeddings_list = [e for e in embeddings_list if e is not None]\n\n if not embeddings_list:\n logger.error(\n \"No valid embeddings available after filtering None values (fail-safe mode). Cannot perform search.\"\n )\n return []\n\n # Create a comprehensive map of model names to embedding objects\n # Check all possible identifiers (deployment, model, model_id, model_name)\n # Also leverage available_models list from EmbeddingsWithModels\n # Handle duplicate identifiers by creating combined keys\n embedding_by_model = {}\n identifier_conflicts = {} # Track which identifiers have conflicts\n\n for idx, emb_obj in enumerate(embeddings_list):\n # Get all possible identifiers for this embedding\n identifiers = []\n deployment = getattr(emb_obj, \"deployment\", None)\n model = getattr(emb_obj, \"model\", None)\n model_id = getattr(emb_obj, \"model_id\", None)\n model_name = getattr(emb_obj, \"model_name\", None)\n dimensions = getattr(emb_obj, \"dimensions\", None)\n available_models_attr = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Embedding object {idx}: deployment={deployment}, model={model}, \"\n f\"model_id={model_id}, model_name={model_name}, dimensions={dimensions}, \"\n f\"available_models={available_models_attr}\"\n )\n\n # If this embedding has available_models dict, map all models to their dedicated instances\n if available_models_attr and isinstance(available_models_attr, dict):\n logger.info(\n f\"Embedding object {idx} provides {len(available_models_attr)} models via available_models dict\"\n )\n for model_name_key, dedicated_embedding in available_models_attr.items():\n if model_name_key and str(model_name_key).strip():\n model_str = str(model_name_key).strip()\n if model_str not in embedding_by_model:\n # Use the dedicated embedding instance from the dict\n embedding_by_model[model_str] = dedicated_embedding\n logger.info(\n f\"Mapped available model '{model_str}' to dedicated embedding instance\"\n )\n else:\n # Conflict detected - track it\n if model_str not in identifier_conflicts:\n identifier_conflicts[model_str] = [embedding_by_model[model_str]]\n identifier_conflicts[model_str].append(dedicated_embedding)\n logger.warning(\n f\"Available model '{model_str}' has conflict - used by multiple embeddings\"\n )\n\n # Also map traditional identifiers (for backward compatibility)\n if deployment:\n identifiers.append(str(deployment))\n if model:\n identifiers.append(str(model))\n if model_id:\n identifiers.append(str(model_id))\n if model_name:\n identifiers.append(str(model_name))\n\n # Map all identifiers to this embedding object\n for identifier in identifiers:\n if identifier not in embedding_by_model:\n embedding_by_model[identifier] = emb_obj\n logger.info(f\"Mapped identifier '{identifier}' to embedding object {idx}\")\n else:\n # Conflict detected - track it\n if identifier not in identifier_conflicts:\n identifier_conflicts[identifier] = [embedding_by_model[identifier]]\n identifier_conflicts[identifier].append(emb_obj)\n logger.warning(\n f\"Identifier '{identifier}' has conflict - used by multiple embeddings\"\n )\n\n # For embeddings with model+deployment, create combined identifier\n # This helps when deployment is the same but model differs\n if deployment and model and deployment != model:\n combined_id = f\"{deployment}:{model}\"\n if combined_id not in embedding_by_model:\n embedding_by_model[combined_id] = emb_obj\n logger.info(\n f\"Created combined identifier '{combined_id}' for embedding object {idx}\"\n )\n\n # Log conflicts\n if identifier_conflicts:\n logger.warning(\n f\"Found {len(identifier_conflicts)} conflicting identifiers. \"\n f\"Consider using combined format 'deployment:model' or specifying unique model names.\"\n )\n for conflict_id, emb_list in identifier_conflicts.items():\n logger.warning(\n f\" Conflict on '{conflict_id}': {len(emb_list)} embeddings use this identifier\"\n )\n\n logger.info(f\"Generating embeddings for {len(available_models)} models in index\")\n logger.info(f\"Available embedding identifiers: {list(embedding_by_model.keys())}\")\n self.log(f\"[SEARCH] Models detected in index: {available_models}\")\n self.log(f\"[SEARCH] Available embedding identifiers: {list(embedding_by_model.keys())}\")\n\n # Track matching status for debugging\n matched_models = []\n unmatched_models = []\n\n for model_name in available_models:\n try:\n # Check if we have an embedding object for this model\n if model_name in embedding_by_model:\n # Use the matching embedding object directly\n emb_obj = embedding_by_model[model_name]\n emb_deployment = getattr(emb_obj, \"deployment\", None)\n emb_model = getattr(emb_obj, \"model\", None)\n emb_model_id = getattr(emb_obj, \"model_id\", None)\n emb_dimensions = getattr(emb_obj, \"dimensions\", None)\n emb_available_models = getattr(emb_obj, \"available_models\", None)\n\n logger.info(\n f\"Using embedding object for model '{model_name}': \"\n f\"deployment={emb_deployment}, model={emb_model}, model_id={emb_model_id}, \"\n f\"dimensions={emb_dimensions}\"\n )\n\n # Check if this is a dedicated instance from available_models dict\n if emb_available_models and isinstance(emb_available_models, dict):\n logger.info(\n f\"Model '{model_name}' using dedicated instance from available_models dict \"\n f\"(pre-configured with correct model and dimensions)\"\n )\n\n # Use the embedding instance directly - no model switching needed!\n vec = emb_obj.embed_query(q)\n query_embeddings[model_name] = vec\n matched_models.append(model_name)\n logger.info(\n f\"Generated embedding for model: {model_name} (actual dimensions: {len(vec)})\"\n )\n self.log(f\"[MATCH] Model '{model_name}' - generated {len(vec)}-dim embedding\")\n else:\n # No matching embedding found for this model\n unmatched_models.append(model_name)\n logger.warning(\n f\"No matching embedding found for model '{model_name}'. \"\n f\"This model will be skipped. Available identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(\n f\"[NO MATCH] Model '{model_name}' - available: {list(embedding_by_model.keys())}\"\n )\n except (\n RuntimeError,\n ValueError,\n ConnectionError,\n TimeoutError,\n AttributeError,\n KeyError,\n ) as e:\n logger.warning(f\"Failed to generate embedding for {model_name}: {e}\")\n self.log(f\"[ERROR] Embedding generation failed for '{model_name}': {e}\")\n\n # Log summary of model matching\n logger.info(\n f\"Model matching summary: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n self.log(\n f\"[SUMMARY] Model matching: {len(matched_models)} matched, {len(unmatched_models)} unmatched\"\n )\n if unmatched_models:\n self.log(f\"[WARN] Unmatched models in index: {unmatched_models}\")\n\n if not query_embeddings:\n msg = (\n f\"Failed to generate embeddings for any model. \"\n f\"Index has models: {available_models}, but no matching embedding objects found. \"\n f\"Available embedding identifiers: {list(embedding_by_model.keys())}\"\n )\n self.log(f\"[FAIL] Search failed: {msg}\")\n raise ValueError(msg)\n\n index_properties = self._get_index_properties(client)\n legacy_vector_field = getattr(self, \"vector_field\", \"chunk_embedding\")\n\n # Build KNN queries for each model\n embedding_fields: list[str] = []\n knn_queries_with_candidates = []\n knn_queries_without_candidates = []\n\n raw_num_candidates = getattr(self, \"num_candidates\", 1000)\n try:\n num_candidates = int(raw_num_candidates) if raw_num_candidates is not None else 0\n except (TypeError, ValueError):\n num_candidates = 0\n use_num_candidates = num_candidates > 0\n\n for model_name, embedding_vector in query_embeddings.items():\n field_name = get_embedding_field_name(model_name)\n selected_field = field_name\n vector_dim = len(embedding_vector)\n\n # Only use the expected dynamic field - no legacy fallback\n # This prevents dimension mismatches between models\n if not self._is_knn_vector_field(index_properties, selected_field):\n logger.warning(\n f\"Skipping model {model_name}: field '{field_name}' is not mapped as knn_vector. \"\n f\"Documents must be indexed with this embedding model before querying.\"\n )\n self.log(\n f\"[SKIP] Field '{selected_field}' not a knn_vector - skipping model '{model_name}'\"\n )\n continue\n\n # Validate vector dimensions match the field dimensions\n field_dim = self._get_field_dimension(index_properties, selected_field)\n if field_dim is not None and field_dim != vector_dim:\n logger.error(\n f\"Dimension mismatch for model '{model_name}': \"\n f\"Query vector has {vector_dim} dimensions but field '{selected_field}' expects {field_dim}. \"\n f\"Skipping this model to prevent search errors.\"\n )\n self.log(\n f\"[DIM MISMATCH] Model '{model_name}': query={vector_dim} vs field={field_dim} - skipping\"\n )\n continue\n\n logger.info(\n f\"Adding KNN query for model '{model_name}': field='{selected_field}', \"\n f\"query_dims={vector_dim}, field_dims={field_dim or 'unknown'}\"\n )\n embedding_fields.append(selected_field)\n\n base_query = {\n \"knn\": {\n selected_field: {\n \"vector\": embedding_vector,\n \"k\": 50,\n }\n }\n }\n\n if use_num_candidates:\n query_with_candidates = copy.deepcopy(base_query)\n query_with_candidates[\"knn\"][selected_field][\"num_candidates\"] = num_candidates\n else:\n query_with_candidates = base_query\n\n knn_queries_with_candidates.append(query_with_candidates)\n knn_queries_without_candidates.append(base_query)\n\n if not knn_queries_with_candidates:\n # No valid fields found - this can happen when:\n # 1. Index is empty (no documents yet)\n # 2. Embedding model has changed and field doesn't exist yet\n # Return empty results instead of failing\n logger.warning(\n \"No valid knn_vector fields found for embedding models. \"\n \"This may indicate an empty index or missing field mappings. \"\n \"Returning empty search results.\"\n )\n self.log(\n f\"[WARN] No valid KNN queries could be built. \"\n f\"Query embeddings generated: {list(query_embeddings.keys())}, \"\n f\"but no matching knn_vector fields found in index.\"\n )\n return []\n\n # Build exists filter - document must have at least one embedding field\n exists_any_embedding = {\n \"bool\": {\n \"should\": [{\"exists\": {\"field\": f}} for f in set(embedding_fields)],\n \"minimum_should_match\": 1,\n }\n }\n\n # Combine user filters with exists filter\n all_filters = [*filter_clauses, exists_any_embedding]\n\n # Get limit and score threshold\n limit = self._resolve_limit(filter_obj, default_limit=self.number_of_results)\n score_threshold = self._resolve_score_threshold(filter_obj)\n\n # Determine the best aggregation field for filename based on index mapping\n filename_agg_field = self._get_filename_agg_field(index_properties)\n\n # Build multi-model hybrid query\n body = {\n \"query\": {\n \"bool\": {\n \"should\": [\n {\n \"dis_max\": {\n \"tie_breaker\": 0.0, # Take only the best match, no blending\n \"boost\": 0.7, # 70% weight for semantic search\n \"queries\": knn_queries_with_candidates,\n }\n },\n {\n \"multi_match\": {\n \"query\": q,\n \"fields\": [\"text^2\", \"filename^1.5\"],\n \"type\": \"best_fields\",\n \"fuzziness\": \"AUTO\",\n \"boost\": 0.3, # 30% weight for keyword search\n }\n },\n ],\n \"minimum_should_match\": 1,\n \"filter\": all_filters,\n }\n },\n \"aggs\": {\n \"data_sources\": {\"terms\": {\"field\": filename_agg_field, \"size\": 20}},\n \"document_types\": {\"terms\": {\"field\": \"mimetype\", \"size\": 10}},\n \"owners\": {\"terms\": {\"field\": \"owner\", \"size\": 10}},\n \"embedding_models\": {\"terms\": {\"field\": \"embedding_model\", \"size\": 10}},\n },\n \"_source\": [\n \"filename\",\n \"mimetype\",\n \"page\",\n \"text\",\n \"source_url\",\n \"owner\",\n \"embedding_model\",\n \"allowed_users\",\n \"allowed_groups\",\n \"allowed_principals\",\n ],\n \"size\": limit,\n }\n\n if score_threshold is not None:\n body[\"min_score\"] = score_threshold\n\n logger.info(\n f\"Executing multi-model hybrid search with {len(knn_queries_with_candidates)} embedding models: \"\n f\"{list(query_embeddings.keys())}\"\n )\n self.log(\n f\"[EXEC] Executing search with {len(knn_queries_with_candidates)} KNN queries, limit={limit}\"\n )\n self.log(f\"[EXEC] Embedding models used: {list(query_embeddings.keys())}\")\n self.log(f\"[EXEC] KNN fields being queried: {embedding_fields}\")\n\n try:\n resp = client.search(index=self.index_name, body=body, params={\"terminate_after\": 0})\n except RequestError as e:\n error_message = str(e)\n lowered = error_message.lower()\n if use_num_candidates and \"num_candidates\" in lowered:\n logger.warning(\n \"Retrying search without num_candidates parameter due to cluster capabilities\",\n error=error_message,\n )\n fallback_body = copy.deepcopy(body)\n try:\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = (\n knn_queries_without_candidates\n )\n except (KeyError, IndexError, TypeError) as inner_err:\n raise e from inner_err\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n elif \"knn_vector\" in lowered or (\"field\" in lowered and \"knn\" in lowered):\n fallback_vector = next(iter(query_embeddings.values()), None)\n if fallback_vector is None:\n raise\n fallback_field = legacy_vector_field or \"chunk_embedding\"\n logger.warning(\n \"KNN search failed for dynamic fields; falling back to legacy field '%s'.\",\n fallback_field,\n )\n fallback_body = copy.deepcopy(body)\n fallback_body[\"query\"][\"bool\"][\"filter\"] = filter_clauses\n knn_fallback = {\n \"knn\": {\n fallback_field: {\n \"vector\": fallback_vector,\n \"k\": 50,\n }\n }\n }\n if use_num_candidates:\n knn_fallback[\"knn\"][fallback_field][\"num_candidates\"] = num_candidates\n fallback_body[\"query\"][\"bool\"][\"should\"][0][\"dis_max\"][\"queries\"] = [knn_fallback]\n resp = client.search(\n index=self.index_name,\n body=fallback_body,\n params={\"terminate_after\": 0},\n )\n else:\n raise\n hits = resp.get(\"hits\", {}).get(\"hits\", [])\n\n logger.info(f\"Found {len(hits)} results\")\n self.log(f\"[RESULT] Search complete: {len(hits)} results found\")\n\n if len(hits) == 0:\n self.log(\n f\"[EMPTY] Debug info: \"\n f\"models_in_index={available_models}, \"\n f\"matched_models={matched_models}, \"\n f\"knn_fields={embedding_fields}, \"\n f\"filters={len(filter_clauses)} clauses\"\n )\n\n return [\n {\n \"page_content\": hit[\"_source\"].get(\"text\", \"\"),\n \"metadata\": {k: v for k, v in hit[\"_source\"].items() if k != \"text\"},\n \"score\": hit.get(\"_score\"),\n }\n for hit in hits\n ]\n\n def search_documents(self) -> Table:\n\n \"\"\"Search documents and return results as a Table.\n\n This is the main interface method that performs the multi-model search using the\n configured search_query and returns results in Langflow's Table (DataFrame) format\n so downstream Parser components can consume them directly.\n\n Always builds the vector store (triggering ingestion if needed), then performs\n search only if a query is provided.\n\n Returns:\n Table containing search results with text and metadata\n\n Raises:\n Exception: If search operation fails\n \"\"\"\n\n try:\n # Always build/cache the vector store to ensure ingestion happens\n logger.info(f\"Search query: {self.search_query}\")\n if self._cached_vector_store is None:\n self.build_vector_store()\n\n # Only perform search if query is provided\n search_query = (self.search_query or \"\").strip()\n if not search_query:\n self.log(\"No search query provided - ingestion completed, returning empty results\")\n\n return Table(data=[])\n\n # Perform search with the provided query\n raw = self.search(search_query)\n raw_list = [Data(text=hit[\"page_content\"], **hit[\"metadata\"]) for hit in raw]\n return Table(data=raw_list)\n\n except Exception as e:\n self.log(f\"search_documents error: {e}\")\n raise\n\n # -------- dynamic UI handling (auth switch) --------\n async def update_build_config(\n self, build_config: dict, field_value: str, field_name: str | None = None\n ) -> dict:\n \"\"\"Dynamically update component configuration based on field changes.\n\n This method handles real-time UI updates, particularly for authentication\n mode changes that show/hide relevant input fields.\n\n Args:\n build_config: Current component configuration\n field_value: New value for the changed field\n field_name: Name of the field that changed\n\n Returns:\n Updated build configuration with appropriate field visibility\n \"\"\"\n try:\n if field_name == \"auth_mode\":\n mode = (field_value or \"basic\").strip().lower()\n is_basic = mode == \"basic\"\n is_jwt = mode == \"jwt\"\n is_openrag = mode == \"openrag\"\n\n build_config[\"username\"][\"show\"] = is_basic\n build_config[\"password\"][\"show\"] = is_basic\n\n build_config[\"jwt_token\"][\"show\"] = is_jwt\n build_config[\"jwt_header\"][\"show\"] = is_jwt\n build_config[\"bearer_prefix\"][\"show\"] = is_jwt\n\n build_config[\"username\"][\"required\"] = is_basic\n build_config[\"password\"][\"required\"] = is_basic\n\n # build_config[\"jwt_token\"][\"required\"] = is_jwt\n build_config[\"jwt_header\"][\"required\"] = is_jwt\n build_config[\"bearer_prefix\"][\"required\"] = False\n\n # In 'openrag' mode, expose the OPENRAG_* fields up front\n # since they are the only credentials required.\n for openrag_field in (\n \"openrag_ingest_url\",\n \"openrag_ingest_token\",\n \"openrag_ingest_run_id\",\n \"openrag_ingest_batch_size\",\n ):\n if openrag_field in build_config:\n build_config[openrag_field][\"advanced\"] = not is_openrag\n build_config[openrag_field][\"required\"] = (\n is_openrag and openrag_field != \"openrag_ingest_batch_size\"\n )\n\n if is_basic or is_openrag:\n build_config[\"jwt_token\"][\"value\"] = \"\"\n\n return build_config\n\n except (KeyError, ValueError) as e:\n self.log(f\"update_build_config error: {e}\")\n\n return build_config\n"
},
"docs_metadata": {
"_input_type": "TableInput",
@@ -4312,6 +4399,99 @@
"track_in_telemetry": true,
"type": "bool",
"value": false
+ },
+ "openrag_ingest_url": {
+ "_input_type": "StrInput",
+ "advanced": true,
+ "display_name": "OpenRAG Ingest URL",
+ "dynamic": false,
+ "info": "Internal OpenRAG callback URL for backend-owned document indexing.",
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": true,
+ "name": "openrag_ingest_url",
+ "override_skip": false,
+ "placeholder": "",
+ "required": false,
+ "show": false,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_URL",
+ "input_types": [
+ "Text",
+ "Message"
+ ]
+ },
+ "openrag_ingest_token": {
+ "_input_type": "StrInput",
+ "advanced": true,
+ "display_name": "OpenRAG Ingest Token",
+ "dynamic": false,
+ "info": "Short-lived token used only for OpenRAG ingest callbacks.",
+ "input_types": [
+ "Text",
+ "Message"
+ ],
+ "load_from_db": true,
+ "name": "openrag_ingest_token",
+ "override_skip": false,
+ "password": true,
+ "placeholder": "",
+ "required": false,
+ "show": false,
+ "title_case": false,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_TOKEN"
+ },
+ "openrag_ingest_run_id": {
+ "_input_type": "StrInput",
+ "advanced": true,
+ "display_name": "OpenRAG Ingest Run ID",
+ "dynamic": false,
+ "info": "",
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": true,
+ "name": "openrag_ingest_run_id",
+ "override_skip": false,
+ "placeholder": "",
+ "required": false,
+ "show": false,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_RUN_ID",
+ "input_types": [
+ "Text",
+ "Message"
+ ]
+ },
+ "openrag_ingest_batch_size": {
+ "_input_type": "IntInput",
+ "advanced": true,
+ "display_name": "OpenRAG Ingest Batch Size",
+ "dynamic": false,
+ "info": "",
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": false,
+ "name": "openrag_ingest_batch_size",
+ "override_skip": false,
+ "placeholder": "",
+ "required": false,
+ "show": false,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "int",
+ "value": 100
}
},
"tool_mode": false
@@ -7924,6 +8104,465 @@
},
"selected": true,
"type": "genericNode"
+ },
+ {
+ "data": {
+ "id": "TextInput-OpenRAGIngest-url",
+ "node": {
+ "base_classes": [
+ "Message"
+ ],
+ "beta": false,
+ "conditional_paths": [],
+ "custom_fields": {},
+ "description": "Get user text inputs.",
+ "display_name": "Text Input",
+ "documentation": "https://docs.langflow.org/text-input-and-output",
+ "edited": false,
+ "field_order": [
+ "input_value",
+ "use_global_variable"
+ ],
+ "frozen": false,
+ "icon": "type",
+ "last_updated": "2026-03-05T16:40:36.785Z",
+ "legacy": false,
+ "metadata": {
+ "code_hash": "518f16485886",
+ "dependencies": {
+ "dependencies": [
+ {
+ "name": "lfx",
+ "version": null
+ }
+ ],
+ "total_dependencies": 1
+ },
+ "module": "lfx.components.input_output.text.TextInputComponent"
+ },
+ "minimized": false,
+ "output_types": [],
+ "outputs": [
+ {
+ "allows_loop": false,
+ "cache": true,
+ "display_name": "Output Text",
+ "group_outputs": false,
+ "loop_types": null,
+ "method": "text_response",
+ "name": "text",
+ "options": null,
+ "required_inputs": null,
+ "selected": "Message",
+ "tool_mode": true,
+ "types": [
+ "Message"
+ ],
+ "value": "__UNDEFINED__"
+ }
+ ],
+ "pinned": false,
+ "template": {
+ "_frontend_node_flow_id": {
+ "value": "72c3d17c-2dac-4a73-b48a-6518473d7830"
+ },
+ "_frontend_node_folder_id": {
+ "value": "7a54b33c-ad03-4c78-b61b-f95adbb74e17"
+ },
+ "_type": "Component",
+ "code": {
+ "advanced": true,
+ "dynamic": true,
+ "fileTypes": [],
+ "file_path": "",
+ "info": "",
+ "list": false,
+ "load_from_db": false,
+ "multiline": true,
+ "name": "code",
+ "password": false,
+ "placeholder": "",
+ "required": true,
+ "show": true,
+ "title_case": false,
+ "type": "code",
+ "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n"
+ },
+ "input_value": {
+ "_input_type": "MultilineInput",
+ "advanced": false,
+ "ai_enabled": false,
+ "copy_field": false,
+ "display_name": "Text",
+ "dynamic": false,
+ "info": "Text to be passed as input.",
+ "input_types": [
+ "Message"
+ ],
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": true,
+ "multiline": false,
+ "name": "input_value",
+ "override_skip": false,
+ "password": true,
+ "placeholder": "",
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_input": true,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_URL"
+ },
+ "is_refresh": false,
+ "use_global_variable": {
+ "_input_type": "BoolInput",
+ "advanced": true,
+ "display_name": "Use Global Variable",
+ "dynamic": false,
+ "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.",
+ "list": false,
+ "list_add_label": "Add More",
+ "name": "use_global_variable",
+ "override_skip": false,
+ "placeholder": "",
+ "real_time_refresh": true,
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": true,
+ "type": "bool",
+ "value": true
+ }
+ },
+ "tool_mode": false
+ },
+ "showNode": false,
+ "type": "TextInput"
+ },
+ "dragging": false,
+ "id": "TextInput-OpenRAGIngest-url",
+ "measured": {
+ "height": 52,
+ "width": 192
+ },
+ "position": {
+ "x": 2009.318175939116,
+ "y": 1153.252846147518
+ },
+ "selected": false,
+ "type": "genericNode"
+ },
+ {
+ "data": {
+ "id": "TextInput-OpenRAGIngest-token",
+ "node": {
+ "base_classes": [
+ "Message"
+ ],
+ "beta": false,
+ "conditional_paths": [],
+ "custom_fields": {},
+ "description": "Get user text inputs.",
+ "display_name": "Text Input",
+ "documentation": "https://docs.langflow.org/text-input-and-output",
+ "edited": false,
+ "field_order": [
+ "input_value",
+ "use_global_variable"
+ ],
+ "frozen": false,
+ "icon": "type",
+ "last_updated": "2026-03-05T16:40:36.785Z",
+ "legacy": false,
+ "metadata": {
+ "code_hash": "518f16485886",
+ "dependencies": {
+ "dependencies": [
+ {
+ "name": "lfx",
+ "version": null
+ }
+ ],
+ "total_dependencies": 1
+ },
+ "module": "lfx.components.input_output.text.TextInputComponent"
+ },
+ "minimized": false,
+ "output_types": [],
+ "outputs": [
+ {
+ "allows_loop": false,
+ "cache": true,
+ "display_name": "Output Text",
+ "group_outputs": false,
+ "loop_types": null,
+ "method": "text_response",
+ "name": "text",
+ "options": null,
+ "required_inputs": null,
+ "selected": "Message",
+ "tool_mode": true,
+ "types": [
+ "Message"
+ ],
+ "value": "__UNDEFINED__"
+ }
+ ],
+ "pinned": false,
+ "template": {
+ "_frontend_node_flow_id": {
+ "value": "72c3d17c-2dac-4a73-b48a-6518473d7830"
+ },
+ "_frontend_node_folder_id": {
+ "value": "7a54b33c-ad03-4c78-b61b-f95adbb74e17"
+ },
+ "_type": "Component",
+ "code": {
+ "advanced": true,
+ "dynamic": true,
+ "fileTypes": [],
+ "file_path": "",
+ "info": "",
+ "list": false,
+ "load_from_db": false,
+ "multiline": true,
+ "name": "code",
+ "password": false,
+ "placeholder": "",
+ "required": true,
+ "show": true,
+ "title_case": false,
+ "type": "code",
+ "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n"
+ },
+ "input_value": {
+ "_input_type": "MultilineInput",
+ "advanced": false,
+ "ai_enabled": false,
+ "copy_field": false,
+ "display_name": "Text",
+ "dynamic": false,
+ "info": "Text to be passed as input.",
+ "input_types": [
+ "Message"
+ ],
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": true,
+ "multiline": false,
+ "name": "input_value",
+ "override_skip": false,
+ "password": true,
+ "placeholder": "",
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_input": true,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_TOKEN"
+ },
+ "is_refresh": false,
+ "use_global_variable": {
+ "_input_type": "BoolInput",
+ "advanced": true,
+ "display_name": "Use Global Variable",
+ "dynamic": false,
+ "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.",
+ "list": false,
+ "list_add_label": "Add More",
+ "name": "use_global_variable",
+ "override_skip": false,
+ "placeholder": "",
+ "real_time_refresh": true,
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": true,
+ "type": "bool",
+ "value": true
+ }
+ },
+ "tool_mode": false
+ },
+ "showNode": false,
+ "type": "TextInput"
+ },
+ "dragging": false,
+ "id": "TextInput-OpenRAGIngest-token",
+ "measured": {
+ "height": 52,
+ "width": 192
+ },
+ "position": {
+ "x": 2009.318175939116,
+ "y": 1235.252846147518
+ },
+ "selected": false,
+ "type": "genericNode"
+ },
+ {
+ "data": {
+ "id": "TextInput-OpenRAGIngest-run_id",
+ "node": {
+ "base_classes": [
+ "Message"
+ ],
+ "beta": false,
+ "conditional_paths": [],
+ "custom_fields": {},
+ "description": "Get user text inputs.",
+ "display_name": "Text Input",
+ "documentation": "https://docs.langflow.org/text-input-and-output",
+ "edited": false,
+ "field_order": [
+ "input_value",
+ "use_global_variable"
+ ],
+ "frozen": false,
+ "icon": "type",
+ "last_updated": "2026-03-05T16:40:36.785Z",
+ "legacy": false,
+ "metadata": {
+ "code_hash": "518f16485886",
+ "dependencies": {
+ "dependencies": [
+ {
+ "name": "lfx",
+ "version": null
+ }
+ ],
+ "total_dependencies": 1
+ },
+ "module": "lfx.components.input_output.text.TextInputComponent"
+ },
+ "minimized": false,
+ "output_types": [],
+ "outputs": [
+ {
+ "allows_loop": false,
+ "cache": true,
+ "display_name": "Output Text",
+ "group_outputs": false,
+ "loop_types": null,
+ "method": "text_response",
+ "name": "text",
+ "options": null,
+ "required_inputs": null,
+ "selected": "Message",
+ "tool_mode": true,
+ "types": [
+ "Message"
+ ],
+ "value": "__UNDEFINED__"
+ }
+ ],
+ "pinned": false,
+ "template": {
+ "_frontend_node_flow_id": {
+ "value": "72c3d17c-2dac-4a73-b48a-6518473d7830"
+ },
+ "_frontend_node_folder_id": {
+ "value": "7a54b33c-ad03-4c78-b61b-f95adbb74e17"
+ },
+ "_type": "Component",
+ "code": {
+ "advanced": true,
+ "dynamic": true,
+ "fileTypes": [],
+ "file_path": "",
+ "info": "",
+ "list": false,
+ "load_from_db": false,
+ "multiline": true,
+ "name": "code",
+ "password": false,
+ "placeholder": "",
+ "required": true,
+ "show": true,
+ "title_case": false,
+ "type": "code",
+ "value": "from typing import Any\n\nfrom lfx.base.io.text import TextComponent\nfrom lfx.io import BoolInput, MultilineInput, Output\nfrom lfx.schema.message import Message\n\n\nclass TextInputComponent(TextComponent):\n display_name = \"Text Input\"\n description = \"Get user text inputs.\"\n documentation: str = \"https://docs.langflow.org/text-input-and-output\"\n icon = \"type\"\n name = \"TextInput\"\n\n inputs = [\n MultilineInput(\n name=\"input_value\",\n display_name=\"Text\",\n info=\"Text to be passed as input.\",\n ),\n BoolInput(\n name=\"use_global_variable\",\n display_name=\"Use Global Variable\",\n info=\"Enable to select from global variables (shows globe icon). Disables multiline editing.\",\n value=False,\n advanced=True,\n real_time_refresh=True,\n ),\n ]\n outputs = [\n Output(display_name=\"Output Text\", name=\"text\", method=\"text_response\"),\n ]\n\n def update_build_config(self, build_config: dict, field_value: Any, field_name: str | None = None) -> dict:\n if field_name == \"use_global_variable\":\n if field_value:\n # Enable global variable mode: single-line with password masking and globe dropdown\n build_config[\"input_value\"][\"multiline\"] = False\n build_config[\"input_value\"][\"password\"] = True\n else:\n # Default mode: multiline text editing\n build_config[\"input_value\"][\"multiline\"] = True\n build_config[\"input_value\"][\"password\"] = False\n return build_config\n\n def text_response(self) -> Message:\n return Message(\n text=self.input_value,\n )\n"
+ },
+ "input_value": {
+ "_input_type": "MultilineInput",
+ "advanced": false,
+ "ai_enabled": false,
+ "copy_field": false,
+ "display_name": "Text",
+ "dynamic": false,
+ "info": "Text to be passed as input.",
+ "input_types": [
+ "Message"
+ ],
+ "list": false,
+ "list_add_label": "Add More",
+ "load_from_db": true,
+ "multiline": false,
+ "name": "input_value",
+ "override_skip": false,
+ "password": true,
+ "placeholder": "",
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_input": true,
+ "trace_as_metadata": true,
+ "track_in_telemetry": false,
+ "type": "str",
+ "value": "OPENRAG_INGEST_RUN_ID"
+ },
+ "is_refresh": false,
+ "use_global_variable": {
+ "_input_type": "BoolInput",
+ "advanced": true,
+ "display_name": "Use Global Variable",
+ "dynamic": false,
+ "info": "Enable to select from global variables (shows globe icon). Disables multiline editing.",
+ "list": false,
+ "list_add_label": "Add More",
+ "name": "use_global_variable",
+ "override_skip": false,
+ "placeholder": "",
+ "real_time_refresh": true,
+ "required": false,
+ "show": true,
+ "title_case": false,
+ "tool_mode": false,
+ "trace_as_metadata": true,
+ "track_in_telemetry": true,
+ "type": "bool",
+ "value": true
+ }
+ },
+ "tool_mode": false
+ },
+ "showNode": false,
+ "type": "TextInput"
+ },
+ "dragging": false,
+ "id": "TextInput-OpenRAGIngest-run_id",
+ "measured": {
+ "height": 52,
+ "width": 192
+ },
+ "position": {
+ "x": 2009.318175939116,
+ "y": 1317.252846147518
+ },
+ "selected": false,
+ "type": "genericNode"
}
],
"viewport": {
diff --git a/frontend/app/api/[...path]/route.ts b/frontend/app/api/[...path]/route.ts
index 1d0c5a126..361dcd3fe 100644
--- a/frontend/app/api/[...path]/route.ts
+++ b/frontend/app/api/[...path]/route.ts
@@ -1,5 +1,9 @@
import { NextRequest, NextResponse } from "next/server";
+function getRequestId(request: NextRequest): string {
+ return request.headers.get("x-request-id") || crypto.randomUUID();
+}
+
export async function GET(
request: NextRequest,
{ params }: { params: Promise<{ path: string[] }> },
@@ -44,6 +48,8 @@ async function proxyRequest(request: NextRequest, params: { path: string[] }) {
if (backendSSL) {
backendUrl = `https://${backendHost}:8000/${path}${searchParams ? `?${searchParams}` : ""}`;
}
+ const requestId = getRequestId(request);
+ const start = performance.now();
try {
let body: string | ArrayBuffer | undefined = undefined;
@@ -91,6 +97,7 @@ async function proxyRequest(request: NextRequest, params: { path: string[] }) {
}
headers.set(key, value);
}
+ headers.set("x-request-id", requestId);
const init: RequestInit = {
method: request.method,
@@ -102,7 +109,22 @@ async function proxyRequest(request: NextRequest, params: { path: string[] }) {
typeof body === "string" ? body : new Uint8Array(body as ArrayBuffer);
init.body = bodyInit;
}
+ // biome-ignore lint/suspicious/noConsole: Server-side proxy timing is needed for CI diagnostics.
+ console.info("[API Proxy] Request started", {
+ request_id: requestId,
+ method: request.method,
+ path: `/${path}`,
+ });
const response = await fetch(backendUrl, init);
+ const durationMs = Math.round(performance.now() - start);
+ // biome-ignore lint/suspicious/noConsole: Server-side proxy timing is needed for CI diagnostics.
+ console.info("[API Proxy] Request", {
+ request_id: requestId,
+ method: request.method,
+ path: `/${path}`,
+ status_code: response.status,
+ duration_ms: durationMs,
+ });
const responseHeaders = new Headers();
@@ -120,6 +142,7 @@ async function proxyRequest(request: NextRequest, params: { path: string[] }) {
for (const cookie of response.headers.getSetCookie()) {
responseHeaders.append("set-cookie", cookie);
}
+ responseHeaders.set("x-request-id", requestId);
// For streaming responses, pass the body directly without buffering
if (response.body) {
@@ -138,10 +161,17 @@ async function proxyRequest(request: NextRequest, params: { path: string[] }) {
});
}
} catch (error) {
- console.error("Proxy error:", error);
+ const durationMs = Math.round(performance.now() - start);
+ console.error("[API Proxy] Request failed", {
+ request_id: requestId,
+ method: request.method,
+ path: `/${path}`,
+ duration_ms: durationMs,
+ error,
+ });
return NextResponse.json(
{ error: "Failed to proxy request" },
- { status: 500 },
+ { status: 500, headers: { "x-request-id": requestId } },
);
}
}
diff --git a/frontend/app/api/queries/useGetSearchQuery.ts b/frontend/app/api/queries/useGetSearchQuery.ts
index 0ccb88eb3..f2cb045bb 100644
--- a/frontend/app/api/queries/useGetSearchQuery.ts
+++ b/frontend/app/api/queries/useGetSearchQuery.ts
@@ -86,7 +86,7 @@ export const useGetSearchQuery = (
query: string,
queryData?: ParsedQueryData | null,
options?: Omit<
- UseQueryOptions<SearchResult, Error, SearchResult, any[]>,
+ UseQueryOptions<SearchResult, Error, SearchResult, unknown[]>,
"queryKey" | "queryFn"
>,
) => {
diff --git a/frontend/lib/chat-stream-parsers.ts b/frontend/lib/chat-stream-parsers.ts
index 61d78fa80..ab0eeaab7 100644
--- a/frontend/lib/chat-stream-parsers.ts
+++ b/frontend/lib/chat-stream-parsers.ts
@@ -2,6 +2,22 @@ import type { FunctionCall, TokenUsage } from "@/app/chat/_types/types";
type Chunk = Record<string, unknown>;
+function getDeltaText(delta: unknown): string {
+ if (typeof delta === "string") {
+ return delta;
+ }
+ if (delta && typeof delta === "object") {
+ const d = delta as Chunk;
+ if (typeof d.content === "string") {
+ return d.content;
+ }
+ if (typeof d.text === "string") {
+ return d.text;
+ }
+ }
+ return "";
+}
+
export function parseOpenAIChatChunk(
chunk: unknown,
content: { value: string },
@@ -237,7 +253,7 @@ export function parseRealtimeChunk(
}
if (type === "response.output_text.delta") {
- content.value += (c.delta as string) || "";
+ content.value += getDeltaText(c.delta);
return true;
}
@@ -261,13 +277,9 @@ export function parseOpenRAGChunk(
}
if (c.delta) {
- if (typeof c.delta === "string") {
- content.value += c.delta;
- return true;
- }
- const delta = c.delta as Chunk;
- if (delta.text && !delta.content) {
- content.value += delta.text as string;
+ const deltaText = getDeltaText(c.delta);
+ if (deltaText) {
+ content.value += deltaText;
return true;
}
}
diff --git a/frontend/tests/core/onboarding.spec.ts b/frontend/tests/core/onboarding.spec.ts
index d274a09c3..188858337 100644
--- a/frontend/tests/core/onboarding.spec.ts
+++ b/frontend/tests/core/onboarding.spec.ts
@@ -28,9 +28,11 @@ test("can configure OpenAI provider", async ({ page }) => {
await page.getByTestId("send-button").click();
- await expect(page.getByText("Thinking")).toBeVisible();
-
- await expect(page.getByText("OPENRAG-GENERIC-ASSET-001")).toBeVisible({
+ const verificationAnswer = page.getByText("OPENRAG-GENERIC-ASSET-001");
+ await expect(page.getByText("Thinking").or(verificationAnswer)).toBeVisible({
+ timeout: 60000,
+ });
+ await expect(verificationAnswer).toBeVisible({
timeout: 60000,
});
diff --git a/frontend/tests/utils/onboarding.ts b/frontend/tests/utils/onboarding.ts
index 8c03827d0..b239f62fc 100644
--- a/frontend/tests/utils/onboarding.ts
+++ b/frontend/tests/utils/onboarding.ts
@@ -37,6 +37,15 @@ export async function completeOnboarding(
// Go to the base URL (frontend)
await page.goto("/");
+ if (reset) {
+ const response = await page.request.post("/api/onboarding/rollback");
+ if (!response.ok() && response.status() !== 400) {
+ const text = await response.text();
+ throw new Error(`Failed to rollback onboarding: ${text}`);
+ }
+ await page.reload();
+ }
+
// Wait for either onboarding to be complete or onboarding content to be visible
const completedLocator = page.getByTestId("onboarding-completed");
const contentLocator = page.getByTestId("onboarding-content");
@@ -45,7 +54,7 @@ export async function completeOnboarding(
await expect(completedLocator.or(contentLocator)).toBeVisible({
timeout: 15000,
});
- } catch (error) {
+ } catch {
console.log("Neither onboarding state visible, refreshing page...");
await page.reload();
await expect(completedLocator.or(contentLocator)).toBeVisible({
@@ -145,11 +154,13 @@ export async function completeOnboarding(
// Complete this step
await page.getByTestId("onboarding-complete-button").click();
- await expect(page.getByText("Thinking")).toBeVisible();
-
const doneLocator = page.getByText("Done");
const errorLocator = page.getByTestId("onboarding-error");
+ await expect(
+ page.getByText("Thinking").or(doneLocator).or(errorLocator),
+ ).toBeVisible();
+
await expect(doneLocator.or(errorLocator)).toBeVisible({
timeout: isEmbedding ? 120000 : 60000,
});
@@ -179,13 +190,18 @@ export async function completeOnboarding(
await expect(page.getByTestId("user-message").first()).toHaveText(
"What is OpenRAG?",
);
- await expect(page.getByText("Thinking")).toBeVisible();
- await expect(page.getByText("is an open-source package")).toBeVisible({
+ const openRagAnswer = page.getByText("is an open-source package");
+ await expect(page.getByText("Thinking").or(openRagAnswer)).toBeVisible({
+ timeout: 60000,
+ });
+ await expect(openRagAnswer).toBeVisible({
timeout: 60000,
});
// 4. Add your document
- await expect(page.getByText("Lastly, let's add your data.")).toBeVisible();
+ await expect(page.getByText("Lastly, let's add your data.")).toBeVisible({
+ timeout: 30000,
+ });
await page.waitForTimeout(2000);
await expect(page.getByTestId("upload-button")).toBeVisible();
diff --git a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
index fb7191924..b62845b55 100644
--- a/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
+++ b/kubernetes/helm/openrag/templates/backend/backend-dotenv.yaml
@@ -51,6 +51,7 @@ stringData:
# Langflow url
LANGFLOW_URL={{ include "openrag.langflow.url" . | quote }}
+ OPENRAG_BACKEND_INTERNAL_URL={{ include "openrag.backend.url" . | quote }}
# Flow IDs
{{- if .Values.langflow.flows.chatFlowId }}
@@ -219,4 +220,4 @@ stringData:
SERVICE_NAME={{ .Values.backend.serviceName | quote }}
{{- end }}
SESSION_SECRET={{ $sessionSecret | quote }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml b/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml
index 8e78c5fe4..88a6b86a6 100644
--- a/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml
+++ b/kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml
@@ -155,6 +155,11 @@ stringData:
SOURCE_URL={{ .Values.langflow.flowDefaults.sourceUrl | quote }}
ALLOWED_USERS={{ .Values.langflow.flowDefaults.allowedUsers | quote }}
ALLOWED_GROUPS={{ .Values.langflow.flowDefaults.allowedGroups | quote }}
+ ALLOWED_PRINCIPALS={{ .Values.langflow.flowDefaults.allowedPrincipals | quote }}
+ OPENRAG_INGEST_URL={{ .Values.langflow.flowDefaults.openragIngestUrl | quote }}
+ OPENRAG_INGEST_TOKEN={{ .Values.langflow.flowDefaults.openragIngestToken | quote }}
+ OPENRAG_INGEST_RUN_ID={{ .Values.langflow.flowDefaults.openragIngestRunId | quote }}
+ OPENRAG_INGEST_BATCH_SIZE={{ .Values.langflow.flowDefaults.openragIngestBatchSize | quote }}
OPENRAG-QUERY-FILTER={{ .Values.langflow.openragQueryFilter | quote }}
DOCLING_TASK_ID={{ .Values.langflow.flowDefaults.doclingTaskId | quote }}
FILENAME={{ .Values.langflow.flowDefaults.filename | quote }}
@@ -171,4 +176,4 @@ stringData:
LANGFLOW_KEY_RETRY_DELAY={{ .Values.langflow.auth.keyRetryDelay | quote }}
{{- end }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/kubernetes/helm/openrag/values.yaml b/kubernetes/helm/openrag/values.yaml
index 2a8295705..ec17ef5e4 100644
--- a/kubernetes/helm/openrag/values.yaml
+++ b/kubernetes/helm/openrag/values.yaml
@@ -140,6 +140,11 @@ langflow:
sourceUrl: ""
allowedUsers: "[]"
allowedGroups: "[]"
+ allowedPrincipals: "[]"
+ openragIngestUrl: "OPENRAG_INGEST_URL"
+ openragIngestToken: "OPENRAG_INGEST_TOKEN"
+ openragIngestRunId: "OPENRAG_INGEST_RUN_ID"
+ openragIngestBatchSize: "100"
doclingTaskId: "None"
filename: "None"
mimetype: "None"
@@ -172,7 +177,7 @@ langflow:
host: "https://cloud.langfuse.com"
# Variables to expose to flows
- variablesToGetFromEnvironment: "JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,OPENSEARCH_INDEX_NAME,DOCLING_SERVE_URL,DOCLING_TASK_ID,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_APIKEY,WATSONX_URL,WATSONX_PROJECT_ID,OLLAMA_BASE_URL"
+ variablesToGetFromEnvironment: "JWT,OPENRAG-QUERY-FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,OPENSEARCH_INDEX_NAME,DOCLING_SERVE_URL,DOCLING_TASK_ID,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,ALLOWED_PRINCIPALS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_APIKEY,WATSONX_URL,WATSONX_PROJECT_ID,OLLAMA_BASE_URL,OPENRAG_INGEST_URL,OPENRAG_INGEST_TOKEN,OPENRAG_INGEST_RUN_ID,OPENRAG_INGEST_BATCH_SIZE"
# Probes
livenessProbe:
diff --git a/kubernetes/operator/internal/controller/env.go b/kubernetes/operator/internal/controller/env.go
index e08b02a12..56d7eeff5 100644
--- a/kubernetes/operator/internal/controller/env.go
+++ b/kubernetes/operator/internal/controller/env.go
@@ -31,7 +31,7 @@ func NewEnvVarManager() *EnvVarManager {
"LANGFLOW_DATABASE_URL": "sqlite:////app/data/langflow.db",
// Variables to expose to Langflow components
- "LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT": "JWT,OPENRAG_QUERY_FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,OPENSEARCH_INDEX_NAME,DOCLING_SERVE_URL,DOCLING_TASK_ID,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL",
+ "LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT": "JWT,OPENRAG_QUERY_FILTER,OPENSEARCH_PASSWORD,OPENSEARCH_URL,OPENSEARCH_INDEX_NAME,DOCLING_SERVE_URL,DOCLING_TASK_ID,OWNER,OWNER_NAME,OWNER_EMAIL,CONNECTOR_TYPE,DOCUMENT_ID,SOURCE_URL,ALLOWED_USERS,ALLOWED_GROUPS,ALLOWED_PRINCIPALS,FILENAME,MIMETYPE,FILESIZE,SELECTED_EMBEDDING_MODEL,OPENAI_API_KEY,ANTHROPIC_API_KEY,WATSONX_API_KEY,WATSONX_ENDPOINT,WATSONX_PROJECT_ID,OLLAMA_BASE_URL,OPENRAG_INGEST_URL,OPENRAG_INGEST_TOKEN,OPENRAG_INGEST_RUN_ID,OPENRAG_INGEST_BATCH_SIZE",
// Authentication and user management
"LANGFLOW_SKIP_AUTH_AUTO_LOGIN": "true",
@@ -52,22 +52,27 @@ func NewEnvVarManager() *EnvVarManager {
"LANGFLOW_KEY_RETRY_DELAY": "2",
// Flow context defaults
- "JWT": "None",
- "OWNER": "None",
- "OWNER_NAME": "None",
- "OWNER_EMAIL": "None",
- "DOCLING_TASK_ID": "None",
- "CONNECTOR_TYPE": "system",
- "CONNECTOR_TYPE_URL": "url",
- "DOCUMENT_ID": "",
- "SOURCE_URL": "",
- "ALLOWED_USERS": "[]",
- "ALLOWED_GROUPS": "[]",
- "OPENRAG_QUERY_FILTER": "{}",
- "FILENAME": "None",
- "MIMETYPE": "None",
- "FILESIZE": "0",
- "SELECTED_EMBEDDING_MODEL": "",
+ "JWT": "None",
+ "OWNER": "None",
+ "OWNER_NAME": "None",
+ "OWNER_EMAIL": "None",
+ "DOCLING_TASK_ID": "None",
+ "CONNECTOR_TYPE": "system",
+ "CONNECTOR_TYPE_URL": "url",
+ "DOCUMENT_ID": "",
+ "SOURCE_URL": "",
+ "ALLOWED_USERS": "[]",
+ "ALLOWED_GROUPS": "[]",
+ "ALLOWED_PRINCIPALS": "[]",
+ "OPENRAG_QUERY_FILTER": "{}",
+ "OPENRAG_INGEST_URL": "OPENRAG_INGEST_URL",
+ "OPENRAG_INGEST_TOKEN": "OPENRAG_INGEST_TOKEN",
+ "OPENRAG_INGEST_RUN_ID": "OPENRAG_INGEST_RUN_ID",
+ "OPENRAG_INGEST_BATCH_SIZE": "100",
+ "FILENAME": "None",
+ "MIMETYPE": "None",
+ "FILESIZE": "0",
+ "SELECTED_EMBEDDING_MODEL": "",
// OpenSearch defaults (for variables in LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT)
"OPENSEARCH_PASSWORD": "None",
@@ -89,13 +94,14 @@ func NewEnvVarManager() *EnvVarManager {
},
DefaultOpenRagBEEnvVars: map[string]string{
// Langflow connection
- "LANGFLOW_URL": "http://langflow:7860",
- "LANGFLOW_TIMEOUT": "2400",
- "LANGFLOW_CONNECT_TIMEOUT": "30",
- "LANGFLOW_AUTO_LOGIN": "true",
- "LANGFLOW_KEY_RETRIES": "15",
- "LANGFLOW_KEY_RETRY_DELAY": "2",
- "LANGFLOW_KEY": "",
+ "LANGFLOW_URL": "http://langflow:7860",
+ "OPENRAG_BACKEND_INTERNAL_URL": "http://openrag-be:8000",
+ "LANGFLOW_TIMEOUT": "2400",
+ "LANGFLOW_CONNECT_TIMEOUT": "30",
+ "LANGFLOW_AUTO_LOGIN": "true",
+ "LANGFLOW_KEY_RETRIES": "15",
+ "LANGFLOW_KEY_RETRY_DELAY": "2",
+ "LANGFLOW_KEY": "",
// Backend data paths
"OPENRAG_DATA_PATH": "/app/backend-data",
diff --git a/kubernetes/operator/internal/controller/env_example_test.go b/kubernetes/operator/internal/controller/env_example_test.go
index 85bf4bff2..5de093503 100644
--- a/kubernetes/operator/internal/controller/env_example_test.go
+++ b/kubernetes/operator/internal/controller/env_example_test.go
@@ -36,14 +36,14 @@ func Example_envVarPriority() {
// Build the .env file content
envFileContent := manager.BuildEnvFileContent(mergedEnvVars)
- fmt.Printf("\n.env file would contain %d bytes\n", len(envFileContent))
+ fmt.Printf("\n.env file generated: %t\n", len(envFileContent) > 0)
// Output:
// LANGFLOW_AUTO_LOGIN: true (from defaults)
// LANGFLOW_WORKERS: 8 (from operator env)
// LANGFLOW_LOG_LEVEL: ERROR (from CR spec)
//
- // .env file would contain 1475 bytes
+ // .env file generated: true
}
// Example showing how different components use different prefixes
diff --git a/kubernetes/operator/internal/controller/env_test.go b/kubernetes/operator/internal/controller/env_test.go
index 51b2a9150..108b29dce 100644
--- a/kubernetes/operator/internal/controller/env_test.go
+++ b/kubernetes/operator/internal/controller/env_test.go
@@ -207,6 +207,7 @@ func TestEnvVarManager_NewEnvVarManagerDefaults(t *testing.T) {
// Verify Backend defaults
assert.NotNil(t, manager.DefaultOpenRagBEEnvVars)
+ assert.Equal(t, "http://openrag-be:8000", manager.DefaultOpenRagBEEnvVars["OPENRAG_BACKEND_INTERNAL_URL"])
assert.Equal(t, "2400", manager.DefaultOpenRagBEEnvVars["LANGFLOW_TIMEOUT"])
assert.Equal(t, "/app/backend-data", manager.DefaultOpenRagBEEnvVars["OPENRAG_DATA_PATH"])
assert.Equal(t, "/app/openrag-documents", manager.DefaultOpenRagBEEnvVars["OPENRAG_DOCUMENTS_PATH"])
diff --git a/kubernetes/operator/internal/controller/openrag_controller.go b/kubernetes/operator/internal/controller/openrag_controller.go
index bc32d08f4..f72316199 100644
--- a/kubernetes/operator/internal/controller/openrag_controller.go
+++ b/kubernetes/operator/internal/controller/openrag_controller.go
@@ -279,6 +279,7 @@ func (r *OpenRAGReconciler) buildBackendEnv(ctx context.Context, o *openragv1alp
// Operator-derived values (always set)
envVars["LANGFLOW_URL"] = "http://" + getServiceName(o, "lf") + ":7860"
+ envVars["OPENRAG_BACKEND_INTERNAL_URL"] = "http://" + getServiceName(o, "be") + ":8000"
// Override with CR-specific configuration
if o.Spec.TenantID != "" {
diff --git a/kubernetes/operator/internal/controller/openrag_controller_test.go b/kubernetes/operator/internal/controller/openrag_controller_test.go
index 746617656..986c3e874 100644
--- a/kubernetes/operator/internal/controller/openrag_controller_test.go
+++ b/kubernetes/operator/internal/controller/openrag_controller_test.go
@@ -236,6 +236,7 @@ func TestReconcile_BackendEnvContainsLangflowURL(t *testing.T) {
envContent = sec.StringData[".env"]
}
assert.Contains(t, envContent, "LANGFLOW_URL=http://"+resourceName("lf")+":7860")
+ assert.Contains(t, envContent, "OPENRAG_BACKEND_INTERNAL_URL=http://"+resourceName("be")+":8000")
}
func TestReconcile_LangflowMountsPVC(t *testing.T) {
@@ -1115,6 +1116,8 @@ func TestReconcile_CustomServiceName_OperatorCreates(t *testing.T) {
}
assert.Contains(t, envContent, "LANGFLOW_URL=http://"+resourceName("lf")+":7860",
"Backend env should reference default langflow service")
+ assert.Contains(t, envContent, "OPENRAG_BACKEND_INTERNAL_URL=http://my-backend-svc:8000",
+ "Backend env should reference custom backend service name")
}
func TestReconcile_CustomServiceName_UserManaged(t *testing.T) {
@@ -1221,6 +1224,8 @@ func TestReconcile_CustomServiceName_Langflow_UsedInBackendEnv(t *testing.T) {
}
assert.Contains(t, envContent, "LANGFLOW_URL=http://custom-lf-svc:7860",
"Backend env should reference custom langflow service name")
+ assert.Contains(t, envContent, "OPENRAG_BACKEND_INTERNAL_URL=http://"+resourceName("be")+":8000",
+ "Backend env should reference default backend service")
}
func TestReconcile_AllComponentsWithCustomNames_OperatorCreates(t *testing.T) {
diff --git a/pyproject.toml b/pyproject.toml
index 9884e2556..84f813436 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -79,6 +79,9 @@ openrag = "tui.main:run_tui"
filterwarnings = [
"error::RuntimeWarning",
]
+markers = [
+ "openrag_skip_app_onboard: selected tests manage their own infrastructure clients and do not need the session app onboarding fixture",
+]
[tool.uv]
package = true
@@ -154,4 +157,3 @@ exclude = ["flows/components"]
module = ["api.settings.endpoints", "api.v1.settings"]
disable_error_code = ["return-value"]
-
diff --git a/scripts/ci/run_integration_suite.sh b/scripts/ci/run_integration_suite.sh
new file mode 100755
index 000000000..2174f356c
--- /dev/null
+++ b/scripts/ci/run_integration_suite.sh
@@ -0,0 +1,231 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+suite="${1:-${TEST_SUITE:-core}}"
+container_runtime="${CONTAINER_RUNTIME:-$(command -v docker >/dev/null 2>&1 && echo docker || echo podman)}"
+env_file="${ENV_FILE:-.env}"
+
+if [[ -f "$env_file" ]]; then
+ compose_cmd=("$container_runtime" compose --env-file "$env_file")
+else
+ compose_cmd=("$container_runtime" compose)
+fi
+
+red=$'\033[0;31m'
+purple=$'\033[38;2;119;62;255m'
+yellow=$'\033[1;33m'
+cyan=$'\033[0;36m'
+green=$'\033[0;32m'
+nc=$'\033[0m'
+
+test_result=0
+
+wait_for_url() {
+ local label="$1"
+ local url="$2"
+ local attempts="${3:-60}"
+
+ echo "${yellow}Waiting for ${label}...${nc}"
+ for _ in $(seq 1 "$attempts"); do
+ if curl -s "$url" >/dev/null 2>&1; then
+ return 0
+ fi
+ sleep 2
+ done
+
+ echo "${red}Timed out waiting for ${label} at ${url}${nc}"
+ return 1
+}
+
+test_jwt_opensearch() {
+ echo "${cyan}=== JWT OpenSearch Authentication Test ===${nc}"
+ echo "${yellow}Generating test JWT token...${nc}"
+ test_token="$(uv run python -c 'from utils.logging_config import configure_logging; configure_logging(log_level="CRITICAL"); from src.session_manager import SessionManager, AnonymousUser; sm = SessionManager("test"); print(sm.create_jwt_token(AnonymousUser()).removeprefix("Bearer "))' 2>/dev/null)"
+ if [[ -z "$test_token" ]]; then
+ echo "${red}Failed to generate JWT token${nc}"
+ return 1
+ fi
+
+ echo "${yellow}Testing JWT against OpenSearch...${nc}"
+ response_file="$(mktemp /tmp/jwt-os-diag.XXXXXX)"
+ if ! curl --fail-with-body -k -s \
+ -o "$response_file" \
+ -H "Authorization: Bearer $test_token" \
+ -H "Content-Type: application/json" \
+ https://localhost:9200/documents/_search \
+ -d '{"query":{"match_all":{}}}'; then
+ echo "${red}curl command failed (network error or HTTP 4xx/5xx)${nc}"
+ head -c 400 "$response_file" 2>/dev/null || true
+ rm -f "$response_file"
+ return 1
+ fi
+
+ echo "${green}Success - OpenSearch accepted JWT${nc}"
+ echo "Response preview:"
+ head -c 200 "$response_file" | sed 's/^/ /' || true
+ rm -f "$response_file"
+ echo ""
+}
+
+dump_logs() {
+ echo "${red}=== Tests failed, dumping container logs ===${nc}"
+ echo ""
+ echo "${yellow}=== Langflow logs (last 500 lines) ===${nc}"
+ "$container_runtime" logs langflow 2>&1 | tail -500 || echo "${red}Could not get Langflow logs${nc}"
+ echo ""
+ echo "${yellow}=== Backend logs (last 500 lines) ===${nc}"
+ "$container_runtime" logs openrag-backend 2>&1 | tail -500 || echo "${red}Could not get backend logs${nc}"
+ echo ""
+ echo "${yellow}=== Frontend logs (last 300 lines) ===${nc}"
+ "$container_runtime" logs openrag-frontend 2>&1 | tail -300 || echo "${red}Could not get frontend logs${nc}"
+ echo ""
+ echo "${yellow}=== OpenSearch logs (last 300 lines) ===${nc}"
+ "$container_runtime" logs os 2>&1 | tail -300 || echo "${red}Could not get OpenSearch logs${nc}"
+ echo ""
+}
+
+teardown() {
+ local status=$?
+ if [[ "$status" -ne 0 && "$test_result" -eq 0 ]]; then
+ test_result="$status"
+ fi
+
+ if [[ "$test_result" -ne 0 ]]; then
+ dump_logs || true
+ fi
+
+ echo "${yellow}Tearing down infra${nc}"
+ uv run python scripts/docling_ctl.py stop || true
+ "${compose_cmd[@]}" down -v 2>/dev/null || true
+
+ exit "$test_result"
+}
+trap teardown EXIT
+
+if [[ -z "${OPENSEARCH_PASSWORD:-}" ]]; then
+ echo "${red}OPENSEARCH_PASSWORD is required${nc}"
+ exit 1
+fi
+
+echo "${yellow}Installing test dependencies...${nc}"
+uv sync --group dev
+
+echo "::group::Start Infrastructure"
+echo "${yellow}Cleaning up old containers and volumes...${nc}"
+"${compose_cmd[@]}" down -v 2>/dev/null || true
+
+echo "${yellow}Starting infra for suite '${suite}' with OpenRAG version '${OPENRAG_VERSION:-latest}'${nc}"
+OPENSEARCH_HOST=opensearch "${compose_cmd[@]}" up -d opensearch dashboards langflow openrag-backend openrag-frontend
+
+echo "${cyan}Architecture: $(uname -m), Platform: $(uname -s)${nc}"
+echo "${yellow}Starting docling-serve...${nc}"
+docling_start_failed=0
+docling_start_output="$(uv run python scripts/docling_ctl.py start --port 5001 --timeout 180 2>&1)" || docling_start_failed=1
+echo "$docling_start_output"
+if [[ "$docling_start_failed" = "1" ]]; then
+ echo "${red}ERROR: docling_ctl.py start failed. Output above.${nc}"
+ uv run python scripts/docling_ctl.py status 2>&1 || true
+ exit 1
+fi
+
+docling_endpoint="$(echo "$docling_start_output" | grep "Endpoint:" | awk '{print $2}')"
+if [[ -z "$docling_endpoint" ]]; then
+ echo "${red}WARNING: docling-serve did not report an endpoint. Defaulting to http://localhost:5001${nc}"
+ docling_endpoint="http://localhost:5001"
+fi
+
+echo "${purple}Docling-serve started at ${docling_endpoint}${nc}"
+echo "${yellow}Docling-serve status check:${nc}"
+uv run python scripts/docling_ctl.py status 2>&1 || true
+
+echo "${yellow}Waiting for backend OIDC endpoint...${nc}"
+for i in $(seq 1 60); do
+ if "$container_runtime" exec openrag-backend curl -s http://localhost:8000/.well-known/openid-configuration >/dev/null 2>&1; then
+ break
+ fi
+ if [[ "$i" -eq 60 ]]; then
+ echo "${red}Backend OIDC endpoint was not reachable in time${nc}"
+ exit 1
+ fi
+ sleep 2
+done
+
+echo "${yellow}Fixing JWT key ownership for test runner (host UID $(id -u))...${nc}"
+"$container_runtime" run --rm -v "$(pwd)/keys:/keys" alpine sh -c "chown $(id -u):$(id -g) /keys/private_key.pem /keys/public_key.pem 2>/dev/null; chmod 600 /keys/private_key.pem; chmod 644 /keys/public_key.pem 2>/dev/null" 2>/dev/null || true
+
+echo "${yellow}Waiting for OpenSearch security config to be fully applied...${nc}"
+for i in $(seq 1 60); do
+ if "$container_runtime" logs os 2>&1 | grep -q "Security configuration applied successfully"; then
+ echo "${purple}Security configuration applied${nc}"
+ break
+ fi
+ if [[ "$i" -eq 60 ]]; then
+ echo "${red}OpenSearch security config was not applied in time${nc}"
+ exit 1
+ fi
+ sleep 2
+done
+
+echo "${yellow}Verifying OIDC authenticator is active in OpenSearch...${nc}"
+for i in $(seq 1 30); do
+ authc_config="$(curl -k -s -u "admin:${OPENSEARCH_PASSWORD}" https://localhost:9200/_opendistro/_security/api/securityconfig 2>/dev/null || true)"
+ if echo "$authc_config" | grep -q "openid_auth_domain"; then
+ echo "${purple}OIDC authenticator configured${nc}"
+ echo "$authc_config" | grep -A 5 "openid_auth_domain" || true
+ break
+ fi
+ if [[ "$i" -eq 30 ]]; then
+ echo "${red}OIDC authenticator NOT found or unreachable in time!${nc}"
+ echo "Security config output: $authc_config"
+ exit 1
+ fi
+ sleep 2
+done
+
+wait_for_url "Langflow" "http://localhost:7860/" 60
+wait_for_url "docling-serve at ${docling_endpoint}" "${docling_endpoint}/health" 60
+echo "::endgroup::"
+
+case "$suite" in
+ core)
+ echo "::group::Core Integration Tests"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ echo "${purple} Core Integration Tests${nc}"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ LOG_LEVEL="${LOG_LEVEL:-DEBUG}" \
+ GOOGLE_OAUTH_CLIENT_ID="" \
+ GOOGLE_OAUTH_CLIENT_SECRET="" \
+ OPENSEARCH_HOST=localhost OPENSEARCH_PORT=9200 \
+ LANGFLOW_OPENSEARCH_HOST=opensearch LANGFLOW_OPENSEARCH_PORT=9200 \
+ OPENSEARCH_USERNAME=admin OPENSEARCH_PASSWORD="${OPENSEARCH_PASSWORD}" \
+ DISABLE_STARTUP_INGEST="${DISABLE_STARTUP_INGEST:-true}" \
+ uv run pytest tests/integration/core -vv -s -o log_cli=true --log-cli-level=DEBUG || test_result=1
+ echo "::endgroup::"
+ test_jwt_opensearch || test_result=1
+ ;;
+ sdk-python)
+ wait_for_url "frontend at http://localhost:3000" "http://localhost:3000/" 60
+ echo "::group::SDK Integration Tests (Python)"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ echo "${purple} SDK Integration Tests (Python)${nc}"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ uv pip install -e sdks/python
+ SDK_TESTS_ONLY=true OPENRAG_URL=http://localhost:3000 uv run pytest tests/integration/sdk/ -vv -s || test_result=1
+ echo "::endgroup::"
+ ;;
+ sdk-typescript)
+ wait_for_url "frontend at http://localhost:3000" "http://localhost:3000/" 60
+ echo "::group::SDK Integration Tests (TypeScript)"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ echo "${purple} SDK Integration Tests (TypeScript)${nc}"
+ echo "${cyan}════════════════════════════════════════${nc}"
+ cd sdks/typescript
+ npm install && npm run build && OPENRAG_URL=http://localhost:3000 npm test || test_result=1
+ cd ../..
+ echo "::endgroup::"
+ ;;
+ *)
+ echo "${red}Unknown integration suite: ${suite}${nc}"
+ test_result=1
+ ;;
+esac
diff --git a/securityconfig/roles.yml b/securityconfig/roles.yml
index ba72f75e0..45b45405a 100644
--- a/securityconfig/roles.yml
+++ b/securityconfig/roles.yml
@@ -3,55 +3,44 @@ _meta:
config_version: 2
openrag_user_role:
- description: "DLS: user can read/write docs they own or are allowed on"
+ description: "DLS: user can read docs they own or are allowed on"
cluster_permissions:
- - "indices:data/write/bulk"
- - "indices:data/write/index"
- "indices:data/read/scroll"
- "indices:data/read/scroll/clear"
- "cluster:monitor/*"
- - "cluster:admin/opensearch/notifications/configs/create"
- - "cluster:admin/opensearch/notifications/configs/list"
- - "cluster:admin/opensearch/notifications/configs/get"
- - "cluster:admin/opensearch/notifications/configs/update"
- - "cluster:admin/opensearch/notifications/configs/delete"
- - "cluster:admin/opensearch/alerting/*"
index_permissions:
- index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*"]
allowed_actions:
- - crud
- - create_index
+ - read
- indices:admin/mappings/get
- - indices:admin/mappings/put
- indices:admin/exists
- indices:admin/get
- - indices:admin/refresh
- - indices:data/write/delete/byquery
- - indices:data/write/update/byquery
dls: >
{"bool":{"should":[
{"term":{"owner":"${user.name}"}},
+ {"term":{"owner":"${attr.jwt.email}"}},
{"term":{"allowed_users":"${user.name}"}},
+ {"term":{"allowed_users":"${attr.jwt.email}"}},
+ {"terms":{"allowed_principals":{"index":"openrag_dls_principals","id":"${user.name}","path":"principals"}}},
{"bool":{"must_not":{"exists":{"field":"owner"}}}}
],"minimum_should_match":1}}
+ - index_patterns: ["openrag_dls_principals", "openrag_dls_principals*"]
+ allowed_actions:
+ - read
+ - indices:admin/exists
+ - indices:data/read/get
+ - indices:data/read/search
+ dls: >
+ {"term":{"user_name":"${user.name}"}}
- index_patterns: ["api_keys", "api_keys*"]
allowed_actions:
- - crud
- - create_index
+ - read
- indices:admin/mappings/get
- - indices:admin/mappings/put
- indices:admin/exists
- indices:admin/get
- - indices:admin/refresh
- - indices:data/write/delete/byquery
- - indices:data/write/update/byquery
dls: >
{"bool":{"should":[
{"term":{"user_id":"${user.name}"}},
{"bool":{"must_not":{"exists":{"field":"user_id"}}}}
],"minimum_should_match":1}}
- - index_patterns: [".opendistro-alerting-config"]
- allowed_actions:
- - crud
- - indices:admin/get
diff --git a/src/api/acl.py b/src/api/acl.py
index 2b9180f7c..4961434d7 100644
--- a/src/api/acl.py
+++ b/src/api/acl.py
@@ -18,7 +18,12 @@ async def get_document_acl(
body={
"query": {"term": {"filename": filename}},
"size": 1,
- "_source": ["owner", "allowed_users", "allowed_groups"],
+ "_source": [
+ "owner",
+ "allowed_users",
+ "allowed_groups",
+ "allowed_principal_labels",
+ ],
},
)
hits = response.get("hits", {}).get("hits", [])
@@ -34,5 +39,6 @@ async def get_document_acl(
"owner": source.get("owner"),
"allowed_users": source.get("allowed_users", []),
"allowed_groups": source.get("allowed_groups", []),
+ "allowed_principal_labels": source.get("allowed_principal_labels", []),
}
)
diff --git a/src/api/connectors.py b/src/api/connectors.py
index ea4cab69e..4a933b455 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -326,6 +326,8 @@ async def reconcile_orphans_for_connector_type(
deleted_chunks=deleted,
id_field=id_field,
)
+ if deleted <= 0:
+ return []
return orphan_ids
@@ -333,9 +335,9 @@ class ConnectorSyncBody(BaseModel):
max_files: int | None = None
selected_files: list[Any] | None = None
# When True, ingest ALL files from the connector (bypasses the existing-files gate).
- # Used by direct-sync providers like IBM COS on initial ingest.
+ # Used by direct-sync providers on initial ingest.
sync_all: bool = False
- # When set, only ingest files from these buckets (IBM COS specific).
+ # When set, only ingest files from these buckets.
bucket_filter: list[str] | None = None
# Per-request ingest options from the connector upload UI (overrides saved Knowledge for this sync).
settings: dict[str, Any] | None = None
@@ -612,7 +614,7 @@ async def connector_sync(
)
elif body.sync_all or body.bucket_filter:
# Full ingest: discover and ingest all files (or files from specific buckets).
- # Used by direct-sync providers (IBM COS) on initial ingest or per-bucket sync.
+ # Used by direct-sync providers on initial ingest or per-bucket sync.
logger.info(
"Full connector ingest requested",
connector_type=connector_type,
@@ -938,12 +940,8 @@ async def connector_webhook(
affected_count=len(affected_files),
)
- # Generate JWT token for the user (needed for OpenSearch authentication)
user = session_manager.get_user(connection.user_id)
- if user:
- jwt_token = session_manager.create_jwt_token(user)
- else:
- jwt_token = None
+ jwt_token = user.jwt_token if user else None
# Trigger incremental sync for affected files
task_id = await connector_service.sync_specific_files(
diff --git a/src/api/documents.py b/src/api/documents.py
index bd92234e2..0ecdb2ed1 100644
--- a/src/api/documents.py
+++ b/src/api/documents.py
@@ -21,7 +21,8 @@ async def delete_documents_by_filename_core(
):
"""Shared delete-by-filename logic for v1 and non-v1 endpoints."""
from config.settings import get_index_name
- from utils.opensearch_queries import build_filename_delete_body
+ from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
+ from utils.opensearch_queries import build_filename_query, build_owned_filename_query
normalized_filename = (filename or "").strip()
if not normalized_filename:
@@ -38,20 +39,24 @@ async def delete_documents_by_filename_core(
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
+ index_name = get_index_name()
- # Owner check: only the document owner may delete
- check = await opensearch_client.search(
- index=get_index_name(),
- body={
- "query": {"term": {"filename": normalized_filename}},
- "size": 1,
- "_source": ["owner"],
- },
+ owned_document_ids = await collect_visible_document_ids(
+ opensearch_client,
+ index=index_name,
+ query=build_owned_filename_query(normalized_filename, user_id),
)
- hits = check.get("hits", {}).get("hits", [])
- if hits:
- doc_owner = hits[0]["_source"].get("owner")
- if doc_owner and doc_owner != user_id:
+
+ if not owned_document_ids:
+ visible_check = await opensearch_client.search(
+ index=index_name,
+ body={
+ "query": build_filename_query(normalized_filename),
+ "size": 1,
+ "_source": ["owner"],
+ },
+ )
+ if visible_check.get("hits", {}).get("hits", []):
return (
{
"success": False,
@@ -63,14 +68,24 @@ async def delete_documents_by_filename_core(
403,
)
- delete_query = build_filename_delete_body(normalized_filename)
- result = await opensearch_client.delete_by_query(
- index=get_index_name(),
- body=delete_query,
- conflicts="proceed",
- )
+ return (
+ {
+ "success": False,
+ "deleted_chunks": 0,
+ "filename": normalized_filename,
+ "message": None,
+ "error": "No matching document chunks were deleted. The file may be missing or not deletable in the current user context.",
+ },
+ 404,
+ )
+
+ from config.settings import clients
- deleted_count = result.get("deleted", 0)
+ deleted_count = await delete_document_ids(
+ clients.opensearch,
+ index=index_name,
+ document_ids=owned_document_ids,
+ )
logger.info(
f"Deleted {deleted_count} chunks for filename {normalized_filename}",
user_id=user_id,
@@ -126,14 +141,14 @@ async def delete_chunks_by_document_ids(
document_ids: list[str],
opensearch_client,
index_name: str,
+ write_opensearch_client=None,
field: str = "document_id",
) -> int:
"""Bulk delete OpenSearch chunks by a keyword field. Returns deleted count.
- DLS-safe: enumerate the visible chunk _ids via search, then issue a single
+ DLS-safe: enumerate the visible chunk _ids via search, then issue a trusted
delete per primary id. `delete_by_query` is silently no-opped under DLS
- (returns deleted:N but leaves docs in place — confirmed in the SharePoint
- rename repro debug log).
+ (returns deleted:N but leaves docs in place).
`field` selects which indexed keyword to match against (default: ``document_id``).
Pass ``field="connector_file_id"`` to clean up chunks for a deleted connector
@@ -142,6 +157,7 @@ async def delete_chunks_by_document_ids(
"""
if not document_ids:
return 0
+ from config.settings import clients
from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
chunk_ids = await collect_visible_document_ids(
@@ -149,8 +165,11 @@ async def delete_chunks_by_document_ids(
index=index_name,
query={"terms": {field: document_ids}},
)
+ write_client = write_opensearch_client or clients.opensearch
+ if write_client is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
return await delete_document_ids(
- opensearch_client,
+ write_client,
index=index_name,
document_ids=chunk_ids,
refresh=True,
diff --git a/src/api/langflow_files.py b/src/api/langflow_files.py
index dc848067e..6950fb287 100644
--- a/src/api/langflow_files.py
+++ b/src/api/langflow_files.py
@@ -9,9 +9,9 @@ from pydantic import BaseModel
from dependencies import (
get_current_user,
get_langflow_file_service,
- get_optional_user,
get_session_manager,
get_task_service,
+ require_permission,
)
from session_manager import User
from utils.file_utils import langflow_safe_filename_and_mimetype
@@ -36,8 +36,7 @@ class DeleteFilesBody(BaseModel):
async def upload_user_file(
file: UploadFile = File(...),
langflow_file_service=Depends(get_langflow_file_service),
- session_manager=Depends(get_session_manager),
- user: User | None = Depends(get_optional_user),
+ user: User = Depends(require_permission("knowledge:upload")),
):
"""Upload a file to Langflow's Files API"""
try:
@@ -51,10 +50,8 @@ async def upload_user_file(
)
file_tuple = (upload_filename, content, upload_mimetype)
- jwt_token = user.jwt_token if user else session_manager.get_effective_jwt_token(None, None)
-
logger.debug("Calling langflow_file_service.upload_user_file")
- result = await langflow_file_service.upload_user_file(file_tuple, jwt_token)
+ result = await langflow_file_service.upload_user_file(file_tuple, user.jwt_token)
logger.debug("Upload successful", result=result)
return JSONResponse(result, status_code=201)
except Exception as e:
@@ -69,7 +66,7 @@ async def run_ingestion(
body: RunIngestionBody,
langflow_file_service=Depends(get_langflow_file_service),
session_manager=Depends(get_session_manager),
- user: User = Depends(get_current_user),
+ user: User = Depends(require_permission("knowledge:upload")),
):
"""Run a Langflow ingestion flow"""
if not body.file_paths and not body.file_ids:
@@ -140,7 +137,7 @@ async def upload_and_ingest_user_file(
langflow_file_service=Depends(get_langflow_file_service),
session_manager=Depends(get_session_manager),
task_service=Depends(get_task_service),
- user: User = Depends(get_current_user),
+ user: User = Depends(require_permission("knowledge:upload")),
):
"""Upload and ingest a file via Langflow (async background task)"""
try:
diff --git a/src/api/langflow_ingest.py b/src/api/langflow_ingest.py
new file mode 100644
index 000000000..186249532
--- /dev/null
+++ b/src/api/langflow_ingest.py
@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import Depends, Header, HTTPException
+from pydantic import BaseModel, Field
+
+from dependencies import get_document_index_writer, get_langflow_ingest_token_service
+from services.document_index_writer import DocumentIndexChunk, DocumentIndexWriter
+from services.langflow_ingest_token_service import LangflowIngestTokenService
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class LangflowIngestChunk(BaseModel):
+ id: str
+ text: str
+ vector: list[float] = Field(min_length=1)
+ page: int | None = None
+ metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+class LangflowIngestBatch(BaseModel):
+ ingest_run_id: str
+ batch_id: int = 0
+ final: bool = False
+ chunks: list[LangflowIngestChunk] = Field(default_factory=list)
+
+
+def _extract_ingest_token(
+ authorization: str | None,
+ x_openrag_ingest_token: str | None,
+) -> str:
+ token = x_openrag_ingest_token or authorization or ""
+ token = token.strip()
+ if token.lower().startswith("bearer "):
+ token = token[7:].strip()
+ if not token:
+ raise HTTPException(status_code=401, detail="Missing Langflow ingest token")
+ return token
+
+
+def _authorized_chunk_id(body: LangflowIngestBatch, document_id: str, index: int) -> str:
+ """Build a backend-owned chunk id inside the token's document namespace."""
+ if not document_id:
+ raise HTTPException(status_code=403, detail="Langflow ingest token is missing document id")
+ return f"{document_id}_{body.batch_id}_{index}"
+
+
+async def ingest_langflow_chunks(
+ body: LangflowIngestBatch,
+ authorization: str | None = Header(default=None),
+ x_openrag_ingest_token: str | None = Header(default=None),
+ token_service: LangflowIngestTokenService = Depends(get_langflow_ingest_token_service),
+ writer: DocumentIndexWriter = Depends(get_document_index_writer),
+):
+ token = _extract_ingest_token(authorization, x_openrag_ingest_token)
+ try:
+ context, jti = token_service.validate_token(token)
+ except ValueError as e:
+ raise HTTPException(status_code=401, detail=str(e)) from e
+
+ if body.ingest_run_id != context.ingest_run_id:
+ raise HTTPException(status_code=403, detail="Ingest run mismatch")
+
+ chunks = [
+ DocumentIndexChunk(
+ chunk_id=_authorized_chunk_id(body, context.document_id, index),
+ text=chunk.text,
+ vector=chunk.vector,
+ page=chunk.page,
+ metadata={**chunk.metadata, "langflow_chunk_id": chunk.id},
+ )
+ for index, chunk in enumerate(body.chunks)
+ ]
+ try:
+ result = await writer.index_chunks(context, chunks, final=body.final)
+ except Exception as e:
+ logger.error(
+ "Langflow ingest callback failed",
+ ingest_run_id=body.ingest_run_id,
+ batch_id=body.batch_id,
+ chunk_count=len(chunks),
+ error=str(e),
+ )
+ raise HTTPException(status_code=500, detail=str(e)) from e
+
+ if body.final:
+ token_service.mark_finalized(jti)
+
+ return {
+ "status": "ok",
+ "batch_id": body.batch_id,
+ **result,
+ }
diff --git a/src/api/settings/endpoints.py b/src/api/settings/endpoints.py
index b8000a360..875b03c5c 100644
--- a/src/api/settings/endpoints.py
+++ b/src/api/settings/endpoints.py
@@ -1327,18 +1327,26 @@ async def rollback_onboarding(
opensearch_client = session_manager.get_user_opensearch_client(
user.user_id, user.jwt_token
)
- from config.settings import get_index_name
+ from config.settings import clients, get_index_name
+ from utils.opensearch_delete import (
+ collect_visible_document_ids,
+ delete_document_ids,
+ )
from utils.opensearch_queries import (
- build_filename_delete_body,
+ build_owned_filename_query,
)
- delete_query = build_filename_delete_body(filename)
- result = await opensearch_client.delete_by_query(
- index=get_index_name(),
- body=delete_query,
- conflicts="proceed",
+ index_name = get_index_name()
+ document_ids = await collect_visible_document_ids(
+ opensearch_client,
+ index=index_name,
+ query=build_owned_filename_query(filename, user.user_id),
+ )
+ deleted_count = await delete_document_ids(
+ clients.opensearch,
+ index=index_name,
+ document_ids=document_ids,
)
- deleted_count = result.get("deleted", 0)
if deleted_count > 0:
deleted_files.append(filename)
logger.info(
diff --git a/src/api/v1/chat.py b/src/api/v1/chat.py
index 328738e1d..50c4da162 100644
--- a/src/api/v1/chat.py
+++ b/src/api/v1/chat.py
@@ -6,9 +6,10 @@ Uses API key authentication. Routes through Langflow (chat_service.langflow_chat
"""
import json
+import time
from typing import Any
-from fastapi import Depends, HTTPException
+from fastapi import Depends, HTTPException, Request
from fastapi.responses import JSONResponse, StreamingResponse
from pydantic import BaseModel
@@ -110,6 +111,7 @@ async def _transform_stream_to_sse(raw_stream, chat_id_container: dict):
async def chat_create_endpoint(
body: ChatV1Body,
+ request: Request,
chat_service=Depends(get_chat_service),
session_manager=Depends(get_session_manager),
user: User = Depends(require_api_key_permission("chat:use")),
@@ -122,6 +124,7 @@ async def chat_create_endpoint(
user_id = user.user_id
storage_user_id = _openrag_user_id(user)
jwt_token = user.jwt_token
+ request_id = request.headers.get("x-request-id")
if body.chat_id:
from api.chat import _assert_owns
@@ -132,6 +135,15 @@ async def chat_create_endpoint(
set_search_limit(body.limit)
set_score_threshold(body.score_threshold)
set_auth_context(user_id, jwt_token)
+ start = time.perf_counter()
+
+ logger.info(
+ "[CHAT V1] Request started",
+ request_id=request_id,
+ stream=body.stream,
+ has_chat_id=bool(body.chat_id),
+ filter_id=body.filter_id,
+ )
if body.stream:
raw_stream = await chat_service.langflow_chat(
@@ -146,6 +158,11 @@ async def chat_create_endpoint(
owner_email=user.email,
storage_user_id=storage_user_id,
)
+ logger.info(
+ "[CHAT V1] Stream initialized",
+ request_id=request_id,
+ duration_ms=round((time.perf_counter() - start) * 1000),
+ )
chat_id_container: dict[str, str] = {}
return StreamingResponse(
_transform_stream_to_sse(raw_stream, chat_id_container),
@@ -169,6 +186,13 @@ async def chat_create_endpoint(
owner_email=user.email,
storage_user_id=storage_user_id,
)
+ logger.info(
+ "[CHAT V1] Request completed",
+ request_id=request_id,
+ duration_ms=round((time.perf_counter() - start) * 1000),
+ response_id=result.get("response_id"),
+ source_count=len(result.get("sources", [])),
+ )
return JSONResponse(
{
"response": result.get("response", ""),
diff --git a/src/app/container.py b/src/app/container.py
index be0a43083..2b32ef0cb 100644
--- a/src/app/container.py
+++ b/src/app/container.py
@@ -18,11 +18,15 @@ from connectors.service import ConnectorService
from services.api_key_service import APIKeyService
from services.auth_service import AuthService
from services.chat_service import ChatService
+from services.dls_principal_service import DLSPrincipalService
from services.docling_polling_service import DoclingPollingService
+from services.document_index_writer import DocumentIndexWriter
from services.document_service import DocumentService
from services.flows_service import FlowsService
+from services.group_acl_service import GroupACLService
from services.knowledge_filter_service import KnowledgeFilterService
from services.langflow_file_service import LangflowFileService
+from services.langflow_ingest_token_service import LangflowIngestTokenService
from services.langflow_mcp_service import LangflowMCPService
from services.models_service import ModelsService
from services.monitor_service import MonitorService
@@ -62,10 +66,13 @@ async def initialize_services():
session_manager = SessionManager(SESSION_SECRET)
models_service = ModelsService()
+ document_index_writer = DocumentIndexWriter()
+ langflow_ingest_token_service = LangflowIngestTokenService()
document_service = DocumentService(
session_manager=session_manager,
models_service=models_service,
docling_service=clients.docling_service,
+ document_index_writer=document_index_writer,
)
search_service = SearchService(session_manager, models_service)
register_search_service(search_service)
@@ -96,6 +103,8 @@ async def initialize_services():
langflow_file_service = LangflowFileService(
flows_service=flows_service,
docling_service=clients.docling_service,
+ document_index_writer=document_index_writer,
+ ingest_token_service=langflow_ingest_token_service,
)
langflow_mcp_service = LangflowMCPService()
@@ -111,6 +120,8 @@ async def initialize_services():
flows_service=flows_service,
langflow_service=langflow_file_service,
)
+ group_acl_service = GroupACLService(connector_service)
+ dls_principal_service = DLSPrincipalService(connector_service)
auth_service = AuthService(
session_manager,
@@ -193,8 +204,12 @@ async def initialize_services():
"chat_service": chat_service,
"flows_service": flows_service,
"langflow_file_service": langflow_file_service,
+ "document_index_writer": document_index_writer,
+ "langflow_ingest_token_service": langflow_ingest_token_service,
"auth_service": auth_service,
"connector_service": connector_service,
+ "group_acl_service": group_acl_service,
+ "dls_principal_service": dls_principal_service,
"knowledge_filter_service": knowledge_filter_service,
"models_service": models_service,
"monitor_service": monitor_service,
diff --git a/src/app/middleware.py b/src/app/middleware.py
index f453499df..5323c2b33 100644
--- a/src/app/middleware.py
+++ b/src/app/middleware.py
@@ -44,6 +44,8 @@ class RequestLoggingMiddleware:
status_code = 500
start = time.perf_counter()
+ if path == "/v1/chat":
+ logger.info("[API] Request started")
async def send_wrapper(message):
nonlocal status_code
diff --git a/src/app/routes/internal.py b/src/app/routes/internal.py
index f55f77e83..1605f88a1 100644
--- a/src/app/routes/internal.py
+++ b/src/app/routes/internal.py
@@ -18,6 +18,7 @@ from api import (
flows,
knowledge_filter,
langflow_files,
+ langflow_ingest,
models,
nudges,
oidc,
@@ -71,6 +72,12 @@ def register_internal_routes(app: FastAPI):
methods=["POST"],
tags=["internal"],
)
+ app.add_api_route(
+ "/internal/ingest/chunks",
+ langflow_ingest.ingest_langflow_chunks,
+ methods=["POST"],
+ tags=["internal"],
+ )
# Upload endpoints
app.add_api_route("/upload_context", upload.upload_context, methods=["POST"], tags=["internal"])
diff --git a/src/config/settings.py b/src/config/settings.py
index 0a30be010..3c3680acc 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -2,6 +2,7 @@ import asyncio
import concurrent.futures
import os
import threading
+from typing import Any
import httpx
from agentd.patch import patch_openai_with_mcp
@@ -36,6 +37,7 @@ LANGFLOW_OPENSEARCH_PORT = get_env_int("LANGFLOW_OPENSEARCH_PORT", OPENSEARCH_PO
OPENSEARCH_USERNAME = os.getenv("OPENSEARCH_USERNAME", "admin")
OPENSEARCH_PASSWORD = os.getenv("OPENSEARCH_PASSWORD")
+OPENRAG_FQDN = os.getenv("OPENRAG_FQDN")
LANGFLOW_URL = os.getenv("LANGFLOW_URL", "http://localhost:7860")
# Optional: public URL for browser links (e.g., http://localhost:7860)
LANGFLOW_PUBLIC_URL = os.getenv("LANGFLOW_PUBLIC_URL")
@@ -46,6 +48,10 @@ LANGFLOW_INGEST_FLOW_ID = (
LANGFLOW_URL_INGEST_FLOW_ID = (
os.getenv("LANGFLOW_URL_INGEST_FLOW_ID") or "72c3d17c-2dac-4a73-b48a-6518473d7830"
)
+OPENRAG_BACKEND_INTERNAL_URL = os.getenv(
+ "OPENRAG_BACKEND_INTERNAL_URL",
+ "http://openrag-backend:8000",
+).rstrip("/")
NUDGES_FLOW_ID = os.getenv("NUDGES_FLOW_ID") or "ebc01d31-1976-46ce-a385-b0240327226c"
@@ -55,7 +61,7 @@ LANGFLOW_SUPERUSER = os.getenv("LANGFLOW_SUPERUSER")
LANGFLOW_SUPERUSER_PASSWORD = os.getenv("LANGFLOW_SUPERUSER_PASSWORD")
# Allow explicit key via environment; generation will be skipped if set
LANGFLOW_KEY = os.getenv("LANGFLOW_KEY")
-SESSION_SECRET = os.getenv("SESSION_SECRET", "your-secret-key-change-in-production")
+SESSION_SECRET = os.getenv("SESSION_SECRET") or "your-secret-key-change-in-production"
# Optional explicit JWT signing key. When set (and IBM auth is off),
# RSA keypair generation is skipped. Read here so callers don't poke
# os.environ directly.
@@ -204,6 +210,29 @@ RBAC_CACHE_BACKEND = os.getenv("CACHE_BACKEND", "memory").lower()
# linger for up to this many seconds after a role mutation.
RBAC_PERMISSION_CACHE_TTL_SECONDS = get_env_int("OPENRAG_PERM_CACHE_TTL", 60)
+# TTL (seconds) for cached upstream group memberships used when minting
+# OpenSearch JWTs. Defaults to 0 so group membership changes are resolved per
+# request unless an operator explicitly accepts bounded staleness.
+GROUP_ACL_CACHE_TTL_SECONDS = get_env_int("OPENRAG_GROUP_ACL_CACHE_TTL", 0)
+
+# Minimum interval (seconds) between DLS principal lookup-index refreshes for
+# the same effective OpenSearch user. This bounds connector directory calls and
+# lookup-index writes on authenticated request paths. Group/alias changes can be
+# stale for up to this many seconds; set to 0 for strict per-request refresh.
+DLS_PRINCIPAL_REFRESH_TTL_SECONDS = get_env_int("OPENRAG_DLS_PRINCIPAL_REFRESH_TTL", 60)
+
+ACL_PRINCIPAL_LABELS_MAPPING = {
+ "type": "object",
+ "properties": {
+ "principal": {"type": "keyword"},
+ "kind": {"type": "keyword"},
+ "provider": {"type": "keyword"},
+ "display_name": {"type": "keyword"},
+ "email": {"type": "keyword"},
+ "external_id": {"type": "keyword"},
+ },
+}
+
# TTL (seconds) for the in-process JWT claims cache. A cached entry is also
# checked against the token's own `exp` claim on every hit, so a revoked token
# can linger at most min(this value, token_remaining_lifetime) seconds.
@@ -283,6 +312,22 @@ LANGFLOW_REQUEST_RETRIES = get_env_int("LANGFLOW_REQUEST_RETRIES", 2)
# Should be >= LANGFLOW_TIMEOUT to allow long-running ingestion to complete
# Default: 3600 seconds (60 minutes)
INGESTION_TIMEOUT = get_env_int("INGESTION_TIMEOUT", 3600)
+LANGFLOW_INGEST_CALLBACK_TTL_SECONDS = get_env_int(
+ "LANGFLOW_INGEST_CALLBACK_TTL_SECONDS",
+ INGESTION_TIMEOUT + 300,
+)
+LANGFLOW_INGEST_CALLBACK_BATCH_SIZE = get_env_int("LANGFLOW_INGEST_CALLBACK_BATCH_SIZE", 100)
+
+OPENSEARCH_JWT_TTL_BUFFER_SECONDS = 300
+
+
+def get_opensearch_jwt_ttl_seconds() -> int:
+ """Return the effective short-lived OpenSearch JWT TTL."""
+ return get_env_int(
+ "OPENRAG_OPENSEARCH_JWT_TTL",
+ INGESTION_TIMEOUT + OPENSEARCH_JWT_TTL_BUFFER_SECONDS,
+ )
+
# Two-phase ingestion: backend-side Docling polling configuration.
# Controls how the OpenRAG backend waits for Docling Serve to finish converting
@@ -344,9 +389,13 @@ INDEX_BODY = {
"embedding_model": {"type": "keyword"},
"source_url": {"type": "keyword"},
"connector_type": {"type": "keyword"},
+ "ingest_run_id": {"type": "keyword"},
+ "connector_file_id": {"type": "keyword"},
"owner": {"type": "keyword"},
"allowed_users": {"type": "keyword"},
"allowed_groups": {"type": "keyword"},
+ "allowed_principals": {"type": "keyword"},
+ "allowed_principal_labels": ACL_PRINCIPAL_LABELS_MAPPING,
"user_permissions": {"type": "object"},
"group_permissions": {"type": "object"},
"created_time": {"type": "date"},
@@ -357,6 +406,24 @@ INDEX_BODY = {
},
}
+DLS_PRINCIPAL_INDEX_NAME = "openrag_dls_principals"
+DLS_PRINCIPAL_INDEX_BODY: dict[str, Any] = {
+ "settings": {
+ "index": {"number_of_replicas": 0, "number_of_shards": 1},
+ },
+ "mappings": {
+ "properties": {
+ "user_name": {"type": "keyword"},
+ "auth_user_id": {"type": "keyword"},
+ "auth_email": {"type": "keyword"},
+ "provider": {"type": "keyword"},
+ "principals": {"type": "keyword"},
+ "principal_labels": ACL_PRINCIPAL_LABELS_MAPPING,
+ "updated_at": {"type": "date"},
+ }
+ },
+}
+
# API Keys index for public API authentication
API_KEYS_INDEX_NAME = "api_keys"
API_KEYS_INDEX_BODY = {
@@ -367,7 +434,7 @@ API_KEYS_INDEX_BODY = {
"mappings": {
"properties": {
"key_id": {"type": "keyword"},
- "key_hash": {"type": "keyword"}, # SHA-256 hash, never store plaintext
+ "key_hash": {"type": "keyword"}, # Keyed digest, never store plaintext
"key_prefix": {"type": "keyword"}, # First 8 chars for display (e.g., "orag_abc1")
"user_id": {"type": "keyword"},
"user_email": {"type": "keyword"},
@@ -1124,6 +1191,22 @@ class AppClients:
retry_on_timeout=True,
)
+ def create_basic_opensearch_client(self, username: str, password: str):
+ """Create an OpenSearch client with explicit basic credentials."""
+ return AsyncOpenSearch(
+ hosts=[{"host": OPENSEARCH_HOST, "port": OPENSEARCH_PORT}],
+ connection_class=AIOHttpConnection,
+ scheme="https",
+ use_ssl=True,
+ verify_certs=False,
+ ssl_assert_fingerprint=None,
+ http_auth=(username, password),
+ http_compress=True,
+ timeout=30,
+ max_retries=3,
+ retry_on_timeout=True,
+ )
+
def create_user_opensearch_client(self, jwt_token: str):
"""Create OpenSearch client with user's auth token."""
return self.create_opensearch_client_from_jwt(jwt_token)
diff --git a/src/connectors/base.py b/src/connectors/base.py
index 7f64c6dbe..492dd6208 100644
--- a/src/connectors/base.py
+++ b/src/connectors/base.py
@@ -1,8 +1,8 @@
+import os
from abc import ABC, abstractmethod
-from typing import Dict, List, Any, Optional, AsyncGenerator
from dataclasses import dataclass
from datetime import datetime
-import os
+from typing import Any
@dataclass
@@ -10,14 +10,20 @@ class DocumentACL:
"""Access Control List information for a document"""
owner: str = None
- allowed_users: List[str] = None
- allowed_groups: List[str] = None
+ allowed_users: list[str] = None
+ allowed_groups: list[str] = None
+ allowed_principals: list[str] = None
+ allowed_principal_labels: list[dict[str, Any]] = None
def __post_init__(self):
if self.allowed_users is None:
self.allowed_users = []
if self.allowed_groups is None:
self.allowed_groups = []
+ if self.allowed_principals is None:
+ self.allowed_principals = []
+ if self.allowed_principal_labels is None:
+ self.allowed_principal_labels = []
@dataclass
@@ -32,7 +38,7 @@ class ConnectorDocument:
acl: DocumentACL
modified_time: datetime
created_time: datetime
- metadata: Dict[str, Any] = None
+ metadata: dict[str, Any] = None
def __post_init__(self):
if self.metadata is None:
@@ -51,22 +57,18 @@ class BaseConnector(ABC):
CONNECTOR_DESCRIPTION: str = None
CONNECTOR_ICON: str = None # Icon identifier or emoji
- def __init__(self, config: Dict[str, Any]):
+ def __init__(self, config: dict[str, Any]):
self.config = config
self._authenticated = False
def get_client_id(self) -> str:
"""Get the OAuth client ID from environment variable"""
if not self.CLIENT_ID_ENV_VAR:
- raise NotImplementedError(
- f"{self.__class__.__name__} must define CLIENT_ID_ENV_VAR"
- )
+ raise NotImplementedError(f"{self.__class__.__name__} must define CLIENT_ID_ENV_VAR")
client_id = os.getenv(self.CLIENT_ID_ENV_VAR)
if not client_id:
- raise ValueError(
- f"Environment variable {self.CLIENT_ID_ENV_VAR} is not set"
- )
+ raise ValueError(f"Environment variable {self.CLIENT_ID_ENV_VAR} is not set")
return client_id
@@ -79,9 +81,7 @@ class BaseConnector(ABC):
secret = os.getenv(self.CLIENT_SECRET_ENV_VAR)
if not secret:
- raise ValueError(
- f"Environment variable {self.CLIENT_SECRET_ENV_VAR} is not set"
- )
+ raise ValueError(f"Environment variable {self.CLIENT_SECRET_ENV_VAR} is not set")
return secret
@@ -96,7 +96,9 @@ class BaseConnector(ABC):
pass
@abstractmethod
- async def list_files(self, page_token: Optional[str] = None, max_files: Optional[int] = None) -> Dict[str, Any]:
+ async def list_files(
+ self, page_token: str | None = None, max_files: int | None = None, **kwargs: Any
+ ) -> dict[str, Any]:
"""List all files. Returns files and next_page_token if any."""
pass
@@ -106,21 +108,21 @@ class BaseConnector(ABC):
pass
@abstractmethod
- async def handle_webhook(self, payload: Dict[str, Any]) -> List[str]:
+ async def handle_webhook(self, payload: dict[str, Any]) -> list[str]:
"""Handle webhook notification. Returns list of affected file IDs."""
pass
def handle_webhook_validation(
- self, request_method: str, headers: Dict[str, str], query_params: Dict[str, str]
- ) -> Optional[str]:
+ self, request_method: str, headers: dict[str, str], query_params: dict[str, str]
+ ) -> str | None:
"""Handle webhook validation (e.g., for subscription setup).
Returns validation response if applicable, None otherwise.
Default implementation returns None (no validation needed)."""
return None
def extract_webhook_channel_id(
- self, payload: Dict[str, Any], headers: Dict[str, str]
- ) -> Optional[str]:
+ self, payload: dict[str, Any], headers: dict[str, str]
+ ) -> str | None:
"""Extract channel/subscription ID from webhook payload/headers.
Must be implemented by each connector."""
raise NotImplementedError(
@@ -136,10 +138,43 @@ class BaseConnector(ABC):
def is_authenticated(self) -> bool:
return self._authenticated
- async def _detect_base_url(self) -> Optional[str]:
+ async def get_current_user_principal_labels(self) -> list[dict[str, Any]]:
+ """Return non-authoritative display labels for current-user ACL principals."""
+ return []
+
+ async def _detect_base_url(self) -> str | None:
"""Auto-detect base URL for the connector.
-
+
Default implementation returns None.
Subclasses (OneDrive, SharePoint) should override this method.
"""
return None
+
+ async def get_current_user_group_roles(self) -> list[str]:
+ """Return OpenSearch backend roles for the current connector user.
+
+ Connectors that support upstream group ACLs can override this hook.
+ The core ACL service calls it generically so new connectors only need
+ to implement their own provider-specific group lookup.
+ """
+ return []
+
+ async def get_current_user_principals(self) -> list[str]:
+ """Return provider-scoped ACL principals for the current connector user.
+
+ Connectors that store user ACLs in provider-specific identity spaces can
+ override this hook. The DLS principal service calls it generically so new
+ connectors only need to provide their own alias resolution.
+ """
+ return []
+
+ @classmethod
+ def get_auth_user_principals(cls, user: Any) -> list[str]:
+ """Return connector principals derivable from the OpenRAG auth user.
+
+ This hook covers cases where a document ACL names a provider user alias
+ but the current OpenRAG user has no saved connector connection to query.
+ Connectors should only return aliases when the auth provider gives enough
+ information to construct the same principal used during ingestion.
+ """
+ return []
diff --git a/src/connectors/connection_manager.py b/src/connectors/connection_manager.py
index 79695bf45..fcc127bcb 100644
--- a/src/connectors/connection_manager.py
+++ b/src/connectors/connection_manager.py
@@ -1,21 +1,23 @@
import json
import os
import uuid
-import aiofiles
-from typing import Dict, List, Any, Optional
+from dataclasses import asdict, dataclass
from datetime import datetime
-from dataclasses import dataclass, asdict
from pathlib import Path
-from utils.logging_config import get_logger
+from typing import Any
-logger = get_logger(__name__)
+import aiofiles
+
+from utils.logging_config import get_logger
+from .aws_s3 import S3Connector
from .base import BaseConnector
from .google_drive import GoogleDriveConnector
-from .sharepoint import SharePointConnector
-from .onedrive import OneDriveConnector
from .ibm_cos import IBMCOSConnector
-from .aws_s3 import S3Connector
+from .onedrive import OneDriveConnector
+from .sharepoint import SharePointConnector
+
+logger = get_logger(__name__)
@dataclass
@@ -25,10 +27,10 @@ class ConnectionConfig:
connection_id: str
connector_type: str # "google_drive", "box", etc.
name: str # User-friendly name
- config: Dict[str, Any] # Connector-specific config
- user_id: Optional[str] = None # For multi-tenant support
+ config: dict[str, Any] # Connector-specific config
+ user_id: str | None = None # For multi-tenant support
created_at: datetime = None
- last_sync: Optional[datetime] = None
+ last_sync: datetime | None = None
is_active: bool = True
def __post_init__(self):
@@ -39,7 +41,7 @@ class ConnectionConfig:
class ConnectionManager:
"""Manages multiple connector connections with persistence"""
- def __init__(self, connections_file: Optional[str] = None):
+ def __init__(self, connections_file: str | None = None):
if connections_file is None:
data_dir = Path(os.getenv("OPENRAG_DATA_PATH", "data"))
self.connections_file = data_dir / "connections.json"
@@ -47,8 +49,8 @@ class ConnectionManager:
self.connections_file = Path(connections_file)
# Ensure data directory exists
self.connections_file.parent.mkdir(parents=True, exist_ok=True)
- self.connections: Dict[str, ConnectionConfig] = {}
- self.active_connectors: Dict[str, BaseConnector] = {}
+ self.connections: dict[str, ConnectionConfig] = {}
+ self.active_connectors: dict[str, BaseConnector] = {}
async def load_connections(self):
"""Load connections from persistent storage"""
@@ -72,7 +74,7 @@ class ConnectionManager:
}
if self.connections_file.exists():
- async with aiofiles.open(self.connections_file, "r") as f:
+ async with aiofiles.open(self.connections_file) as f:
data = json.loads(await f.read())
for conn_data in data.get("connections", []):
@@ -160,8 +162,8 @@ class ConnectionManager:
await f.write(json.dumps(data, indent=2))
async def _get_existing_connection(
- self, connector_type: str, user_id: Optional[str] = None
- ) -> Optional[ConnectionConfig]:
+ self, connector_type: str, user_id: str | None = None
+ ) -> ConnectionConfig | None:
"""Find existing active connection for the same connector type and user"""
for connection in self.connections.values():
if (
@@ -250,7 +252,7 @@ class ConnectionManager:
)
# Remove or deactivate duplicate connections
- for connection_id, connection in connections_to_remove:
+ for connection_id, _connection in connections_to_remove:
if remove_duplicates:
await self.delete_connection(connection_id) # Handles token cleanup
else:
@@ -267,7 +269,7 @@ class ConnectionManager:
connection_id: str,
connector_type: str = None,
name: str = None,
- config: Dict[str, Any] = None,
+ config: dict[str, Any] = None,
user_id: str = None,
) -> bool:
"""Update an existing connection configuration"""
@@ -307,8 +309,8 @@ class ConnectionManager:
self,
connector_type: str,
name: str,
- config: Dict[str, Any],
- user_id: Optional[str] = None,
+ config: dict[str, Any],
+ user_id: str | None = None,
) -> str:
"""Create a new connection configuration, ensuring only one per provider per user"""
@@ -358,8 +360,8 @@ class ConnectionManager:
return connection_id
async def list_connections(
- self, user_id: Optional[str] = None, connector_type: Optional[str] = None
- ) -> List[ConnectionConfig]:
+ self, user_id: str | None = None, connector_type: str | None = None
+ ) -> list[ConnectionConfig]:
"""List connections, optionally filtered by user or connector type"""
connections = list(self.connections.values())
@@ -395,7 +397,7 @@ class ConnectionManager:
try:
if hasattr(connector, "webhook_channel_id") and connector.webhook_channel_id:
await connector.cleanup_subscription(connector.webhook_channel_id)
- except:
+ except Exception:
pass # Best effort cleanup
del self.active_connectors[connection_id]
@@ -404,7 +406,7 @@ class ConnectionManager:
await self.save_connections()
return True
- async def get_connector(self, connection_id: str) -> Optional[BaseConnector]:
+ async def get_connector(self, connection_id: str) -> BaseConnector | None:
"""Get an active connector instance"""
logger.debug(f"Getting connector for connection_id: {connection_id}")
@@ -441,8 +443,8 @@ class ConnectionManager:
return None
def get_available_connector_types(
- self, user_id: Optional[str] = None
- ) -> Dict[str, Dict[str, Any]]:
+ self, user_id: str | None = None
+ ) -> dict[str, dict[str, Any]]:
"""Get available connector types with their metadata.
Availability is user-scoped when ``user_id`` is provided:
@@ -483,7 +485,29 @@ class ConnectionManager:
},
}
- def _has_saved_credentials_for_user(self, connector_type: str, user_id: Optional[str]) -> bool:
+ def get_auth_user_principals(self, user: Any) -> list[str]:
+ """Return connector ACL principals derivable from the OpenRAG auth user."""
+ from utils.group_acl import unique_acl_principals
+
+ principals: list[str] = []
+ for connector_cls in (
+ GoogleDriveConnector,
+ SharePointConnector,
+ OneDriveConnector,
+ IBMCOSConnector,
+ S3Connector,
+ ):
+ try:
+ principals.extend(connector_cls.get_auth_user_principals(user) or [])
+ except Exception as e:
+ logger.debug(
+ "Connector auth-user principal resolver failed",
+ connector=connector_cls.__name__,
+ error=str(e),
+ )
+ return unique_acl_principals(principals)
+
+ def _has_saved_credentials_for_user(self, connector_type: str, user_id: str | None) -> bool:
"""Check if user has an active saved connection with usable credentials."""
for connection in self.connections.values():
if connection.connector_type != connector_type or not connection.is_active:
@@ -499,7 +523,7 @@ class ConnectionManager:
continue
return False
- def _is_connector_available(self, connector_type: str, user_id: Optional[str] = None) -> bool:
+ def _is_connector_available(self, connector_type: str, user_id: str | None = None) -> bool:
"""Check whether connector is available for use by the given user."""
try:
temp_config = ConnectionConfig(
@@ -568,11 +592,11 @@ class ConnectionManager:
return True
return False
- async def get_connection(self, connection_id: str) -> Optional[ConnectionConfig]:
+ async def get_connection(self, connection_id: str) -> ConnectionConfig | None:
"""Get connection configuration"""
return self.connections.get(connection_id)
- async def get_connection_by_webhook_id(self, webhook_id: str) -> Optional[ConnectionConfig]:
+ async def get_connection_by_webhook_id(self, webhook_id: str) -> ConnectionConfig | None:
"""Find a connection by its webhook/subscription ID"""
for connection in self.connections.values():
# Check if the webhook ID is stored in the connection config
@@ -613,8 +637,9 @@ class ConnectionManager:
# Store the subscription and resource IDs in connection config
connection_config.config["webhook_channel_id"] = subscription_id
connection_config.config["subscription_id"] = subscription_id # Alternative field
- if getattr(connector, "webhook_resource_id", None):
- connection_config.config["resource_id"] = connector.webhook_resource_id
+ resource_id = getattr(connector, "webhook_resource_id", None)
+ if resource_id:
+ connection_config.config["resource_id"] = resource_id
# Save updated connection config
await self.save_connections()
@@ -658,8 +683,9 @@ class ConnectionManager:
# Store the subscription and resource IDs in connection config
connection_config.config["webhook_channel_id"] = subscription_id
connection_config.config["subscription_id"] = subscription_id
- if getattr(connector, "webhook_resource_id", None):
- connection_config.config["resource_id"] = connector.webhook_resource_id
+ resource_id = getattr(connector, "webhook_resource_id", None)
+ if resource_id:
+ connection_config.config["resource_id"] = resource_id
# Save updated connection config
await self.save_connections()
diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.py
index 44a5a0a02..ffdb22264 100644
--- a/src/connectors/google_drive/connector.py
+++ b/src/connectors/google_drive/connector.py
@@ -11,9 +11,18 @@ from typing import Any
from googleapiclient.errors import HttpError
from googleapiclient.http import MediaIoBaseDownload
+from utils.group_acl import acl_principal_label, unique_acl_principal_labels
from utils.logging_config import get_logger
from ..base import BaseConnector, ConnectorDocument, DocumentACL
+from ..google_drive_acl import (
+ GOOGLE_DRIVE_GROUP_PROVIDER,
+ get_current_user_google_group_roles,
+ get_current_user_google_principal_labels,
+ get_current_user_google_principals,
+ google_drive_group_role,
+ google_drive_user_principal,
+)
from .oauth import GoogleDriveOAuth
logger = get_logger(__name__)
@@ -82,6 +91,15 @@ class GoogleDriveConnector(BaseConnector):
_FILE_ID_ALIASES = ("file_ids", "selected_file_ids", "selected_files")
_FOLDER_ID_ALIASES = ("folder_ids", "selected_folder_ids", "selected_folders")
+ @classmethod
+ def get_auth_user_principals(cls, user: Any) -> list[str]:
+ """Return Google Drive principals derivable from a Google-auth OpenRAG user."""
+ provider = str(getattr(user, "provider", "") or "").lower()
+ if not provider.startswith("google"):
+ return []
+ principal = google_drive_user_principal(getattr(user, "email", None))
+ return [principal] if principal else []
+
def emit(self, doc: ConnectorDocument) -> None:
"""
Emit a ConnectorDocument instance.
@@ -92,6 +110,8 @@ class GoogleDriveConnector(BaseConnector):
logger.debug(f"Emitting document: {doc.id} ({doc.filename})")
def __init__(self, config: dict[str, Any]) -> None:
+ super().__init__(config)
+
# Read from config OR env (backend env, not NEXT_PUBLIC_*):
env_client_id = os.getenv(self.CLIENT_ID_ENV_VAR)
env_client_secret = os.getenv(self.CLIENT_SECRET_ENV_VAR)
@@ -573,6 +593,24 @@ class GoogleDriveConnector(BaseConnector):
logger.error("[GoogleDrive] authenticate failed: %s", e)
return False
+ async def get_current_user_group_roles(self) -> list[str]:
+ """Return canonical group ACL roles for the connected Google user."""
+ if not self._authenticated and not await self.authenticate():
+ return []
+ return await get_current_user_google_group_roles(self.service, self.creds)
+
+ async def get_current_user_principals(self) -> list[str]:
+ """Return canonical user ACL principals for the connected Google user."""
+ if not self._authenticated and not await self.authenticate():
+ return []
+ return await get_current_user_google_principals(self.service, self.creds)
+
+ async def get_current_user_principal_labels(self) -> list[dict[str, Any]]:
+ """Return display labels for current Google user/group ACL principals."""
+ if not self._authenticated and not await self.authenticate():
+ return []
+ return await get_current_user_google_principal_labels(self.service, self.creds)
+
async def list_files(
self,
page_token: str | None = None,
@@ -641,6 +679,8 @@ class GoogleDriveConnector(BaseConnector):
allowed_users = []
allowed_groups = []
+ allowed_principals = []
+ allowed_principal_labels = []
owner = None
for perm in permissions_list.get("permissions", []):
@@ -658,19 +698,60 @@ class GoogleDriveConnector(BaseConnector):
# Add allowed users
if perm_type == "user" and email:
allowed_users.append(email)
+ user_principal = google_drive_user_principal(email)
+ if user_principal:
+ allowed_principals.append(user_principal)
+ label = acl_principal_label(
+ user_principal,
+ kind="user",
+ provider=GOOGLE_DRIVE_GROUP_PROVIDER,
+ display_name=perm.get("displayName") or email,
+ email=email,
+ external_id=email,
+ )
+ if label:
+ allowed_principal_labels.append(label)
# Add allowed groups
elif perm_type == "group" and email:
- allowed_groups.append(email)
+ group_role = google_drive_group_role(email)
+ if group_role:
+ allowed_groups.append(group_role)
+ allowed_principals.append(group_role)
+ label = acl_principal_label(
+ group_role,
+ kind="group",
+ provider=GOOGLE_DRIVE_GROUP_PROVIDER,
+ display_name=perm.get("displayName") or email,
+ email=email,
+ external_id=email,
+ )
+ if label:
+ allowed_principal_labels.append(label)
# Fallback to file owners if no owner found in permissions
if not owner and file_meta.get("owners"):
owner = file_meta["owners"][0].get("emailAddress")
+ owner_principal = google_drive_user_principal(owner)
+ if owner_principal:
+ allowed_principals.append(owner_principal)
+ label = acl_principal_label(
+ owner_principal,
+ kind="user",
+ provider=GOOGLE_DRIVE_GROUP_PROVIDER,
+ display_name=owner,
+ email=owner,
+ external_id=owner,
+ )
+ if label:
+ allowed_principal_labels.append(label)
return DocumentACL(
owner=owner,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
+ allowed_principals=allowed_principals,
+ allowed_principal_labels=unique_acl_principal_labels(allowed_principal_labels),
)
except Exception as e:
@@ -679,10 +760,21 @@ class GoogleDriveConnector(BaseConnector):
owner = None
if file_meta.get("owners"):
owner = file_meta["owners"][0].get("emailAddress")
+ owner_principal = google_drive_user_principal(owner)
+ owner_label = acl_principal_label(
+ owner_principal,
+ kind="user",
+ provider=GOOGLE_DRIVE_GROUP_PROVIDER,
+ display_name=owner,
+ email=owner,
+ external_id=owner,
+ )
return DocumentACL(
owner=owner,
allowed_users=[owner] if owner else [],
allowed_groups=[],
+ allowed_principals=[owner_principal] if owner_principal else [],
+ allowed_principal_labels=[owner_label] if owner_label else [],
)
async def get_file_content(self, file_id: str) -> ConnectorDocument:
diff --git a/src/connectors/google_drive/oauth.py b/src/connectors/google_drive/oauth.py
index fda96b898..672feba04 100644
--- a/src/connectors/google_drive/oauth.py
+++ b/src/connectors/google_drive/oauth.py
@@ -1,7 +1,6 @@
import asyncio
-import os
import json
-from typing import Optional
+import os
import requests as req_lib
from google.auth.transport.requests import Request
@@ -25,6 +24,8 @@ class GoogleDriveOAuth:
"profile",
"https://www.googleapis.com/auth/drive.readonly",
"https://www.googleapis.com/auth/drive.metadata.readonly",
+ "https://www.googleapis.com/auth/cloud-identity.groups.readonly",
+ "https://www.googleapis.com/auth/admin.directory.group.readonly",
]
AUTH_ENDPOINT = "https://accounts.google.com/o/oauth2/v2/auth"
@@ -32,22 +33,33 @@ class GoogleDriveOAuth:
def __init__(
self,
- client_id: Optional[str] = None,
- client_secret: Optional[str] = None,
+ client_id: str | None = None,
+ client_secret: str | None = None,
token_file: str = "token.json",
):
self.client_id = client_id
self.client_secret = client_secret
self.token_file = token_file
- self.creds: Optional[Credentials] = None
+ self.creds: Credentials | None = None
def _make_timeout_request(self) -> Request:
"""Build a google-auth Request transport with a bounded timeout."""
session = req_lib.Session()
- session.timeout = _REFRESH_TIMEOUT_SECONDS
+ session.timeout = _REFRESH_TIMEOUT_SECONDS # type: ignore[attr-defined]
return Request(session=session)
- async def load_credentials(self) -> Optional[Credentials]:
+ def _missing_required_scopes(self, scopes: list[str] | None) -> list[str]:
+ current_scopes = set(scopes or [])
+ return [scope for scope in self.SCOPES if scope not in current_scopes]
+
+ def _remove_token_file(self) -> None:
+ if os.path.exists(self.token_file):
+ try:
+ os.remove(self.token_file)
+ except Exception:
+ logger.debug("[GoogleDrive] load_credentials: failed to remove token file")
+
+ async def load_credentials(self) -> Credentials | None:
"""Load existing credentials from token file"""
from utils.encryption import read_encrypted_file
@@ -65,7 +77,20 @@ class GoogleDriveOAuth:
os.remove(self.token_file)
return None
- logger.debug("[GoogleDrive] load_credentials: token data loaded, creating Credentials object")
+ logger.debug(
+ "[GoogleDrive] load_credentials: token data loaded, creating Credentials object"
+ )
+
+ missing_scopes = self._missing_required_scopes(token_data.get("scopes"))
+ if missing_scopes:
+ logger.info(
+ "[GoogleDrive] load_credentials: stored token is missing required scopes; "
+ "removing it so the user re-authenticates. missing_scopes=%s",
+ missing_scopes,
+ )
+ self.creds = None
+ self._remove_token_file()
+ return None
self.creds = Credentials(
token=token_data.get("token"),
@@ -100,11 +125,7 @@ class GoogleDriveOAuth:
except Exception as e:
logger.debug("[GoogleDrive] load_credentials: token refresh failed: %s", e)
self.creds = None
- if os.path.exists(self.token_file):
- try:
- os.remove(self.token_file)
- except Exception:
- pass
+ self._remove_token_file()
raise ValueError(
f"Failed to refresh Google Drive credentials. "
f"The refresh token may have expired or been revoked. "
@@ -123,24 +144,24 @@ class GoogleDriveOAuth:
async def save_credentials(self):
"""Save credentials to token file (without client_secret)"""
if self.creds:
+ scopes = getattr(self.creds, "granted_scopes", None) or self.creds.scopes or self.SCOPES
# Create minimal token data without client_secret
token_data = {
"token": self.creds.token,
"refresh_token": self.creds.refresh_token,
"id_token": self.creds.id_token,
- "scopes": self.creds.scopes,
+ "scopes": list(scopes),
}
# Add expiry if available
if self.creds.expiry:
token_data["expiry"] = self.creds.expiry.isoformat()
-
+
from utils.encryption import write_encrypted_file
+
await write_encrypted_file(self.token_file, json.dumps(token_data))
- def create_authorization_url(
- self, redirect_uri: str, state: Optional[str] = None
- ) -> str:
+ def create_authorization_url(self, redirect_uri: str, state: str | None = None) -> str:
"""Create authorization URL for OAuth flow"""
# Create flow from client credentials directly
client_config = {
@@ -152,9 +173,7 @@ class GoogleDriveOAuth:
}
}
- flow = Flow.from_client_config(
- client_config, scopes=self.SCOPES, redirect_uri=redirect_uri
- )
+ flow = Flow.from_client_config(client_config, scopes=self.SCOPES, redirect_uri=redirect_uri)
kwargs = {
"access_type": "offline",
@@ -172,9 +191,7 @@ class GoogleDriveOAuth:
return auth_url
- async def handle_authorization_callback(
- self, authorization_code: str, state: str
- ) -> bool:
+ async def handle_authorization_callback(self, authorization_code: str, state: str) -> bool:
"""Handle OAuth callback and exchange code for tokens"""
if not hasattr(self, "_flow") or self._flow_state != state:
raise ValueError("Invalid OAuth state")
diff --git a/src/connectors/google_drive_acl.py b/src/connectors/google_drive_acl.py
new file mode 100644
index 000000000..be2bb1cba
--- /dev/null
+++ b/src/connectors/google_drive_acl.py
@@ -0,0 +1,408 @@
+"""Google Drive group ACL helpers."""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import jwt
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+from utils.group_acl import (
+ acl_principal_label,
+ canonical_group_role,
+ canonical_user_principal,
+ unique_acl_principal_labels,
+)
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+GOOGLE_DRIVE_GROUP_PROVIDER = "gdrive"
+GOOGLE_GROUP_LABEL = "cloudidentity.googleapis.com/groups.discussion_forum"
+
+
+def _group_tenant(group_email: str) -> str:
+ if "@" in group_email:
+ return group_email.rsplit("@", 1)[1].lower()
+ return "global"
+
+
+def google_drive_group_role(group_email: str | None) -> str | None:
+ """Return the canonical OpenSearch role for a Google Drive group email."""
+ if not group_email:
+ return None
+ email = group_email.strip().lower()
+ if not email:
+ return None
+ return canonical_group_role(
+ GOOGLE_DRIVE_GROUP_PROVIDER,
+ _group_tenant(email),
+ email,
+ )
+
+
+def google_drive_user_principal(user_email: str | None) -> str | None:
+ """Return the canonical DLS principal for a Google Drive user email."""
+ if not user_email:
+ return None
+ email = user_email.strip().lower()
+ if not email:
+ return None
+ return canonical_user_principal(
+ GOOGLE_DRIVE_GROUP_PROVIDER,
+ _group_tenant(email),
+ email,
+ )
+
+
+def _email_from_id_token(id_token: str | None) -> str | None:
+ if not id_token:
+ return None
+ try:
+ claims = jwt.decode(
+ id_token,
+ options={"verify_signature": False, "verify_aud": False},
+ )
+ email = claims.get("email")
+ if email:
+ return str(email)
+ except Exception as e:
+ logger.debug("Could not decode Google id_token email", error=str(e))
+ return None
+
+
+async def _execute_google_request(request: Any) -> dict[str, Any]:
+ return await asyncio.to_thread(request.execute)
+
+
+def _cel_string(value: str) -> str:
+ return value.replace("\\", "\\\\").replace("'", "\\'")
+
+
+async def _get_drive_user_email(drive_service: Any, credentials: Any) -> str | None:
+ email = _email_from_id_token(getattr(credentials, "id_token", None))
+ if email:
+ return email
+
+ if drive_service is None:
+ return None
+
+ try:
+ about = await _execute_google_request(
+ drive_service.about().get(fields="user(emailAddress)")
+ )
+ return about.get("user", {}).get("emailAddress")
+ except Exception as e:
+ logger.warning("Google Drive group ACL lookup could not resolve user email", error=str(e))
+ return None
+
+
+async def _get_cloud_identity_group_roles(
+ credentials: Any,
+ user_email: str,
+) -> list[str] | None:
+ """Fetch current-user groups through Cloud Identity when the tenant allows it."""
+ try:
+ cloud_identity_service = build(
+ "cloudidentity",
+ "v1",
+ credentials=credentials,
+ cache_discovery=False,
+ )
+ except Exception as e:
+ logger.debug("Could not create Google Cloud Identity client for group ACLs", error=str(e))
+ return None
+
+ roles: list[str] = []
+ seen: set[str] = set()
+ page_token: str | None = None
+ query = f"member_key_id == '{_cel_string(user_email)}' && '{GOOGLE_GROUP_LABEL}' in labels"
+
+ try:
+ while True:
+ request = (
+ cloud_identity_service.groups()
+ .memberships()
+ .searchTransitiveGroups(
+ parent="groups/-",
+ query=query,
+ pageSize=200,
+ pageToken=page_token,
+ )
+ )
+ response = await _execute_google_request(request)
+
+ for membership in response.get("memberships", []) or []:
+ group_key = membership.get("groupKey", {}) or {}
+ role = google_drive_group_role(group_key.get("id"))
+ if role and role not in seen:
+ seen.add(role)
+ roles.append(role)
+
+ page_token = response.get("nextPageToken")
+ if not page_token:
+ break
+ except HttpError as e:
+ status = getattr(getattr(e, "resp", None), "status", None)
+ if status in (401, 403):
+ logger.info(
+ "Google Cloud Identity group ACL lookup unavailable; falling back to "
+ "Admin SDK if permitted",
+ status_code=status,
+ )
+ return None
+ logger.warning("Google Cloud Identity group ACL lookup failed", error=str(e))
+ return None
+ except Exception as e:
+ logger.debug("Google Cloud Identity group ACL lookup errored", error=str(e))
+ return None
+
+ return roles
+
+
+async def get_current_user_google_group_roles(
+ drive_service: Any,
+ credentials: Any,
+) -> list[str]:
+ """Fetch Google Workspace groups for the connected Drive user."""
+ if credentials is None:
+ return []
+
+ user_email = await _get_drive_user_email(drive_service, credentials)
+ if not user_email:
+ return []
+
+ cloud_identity_roles = await _get_cloud_identity_group_roles(credentials, user_email)
+ if cloud_identity_roles is not None:
+ return cloud_identity_roles
+
+ try:
+ directory_service = build(
+ "admin",
+ "directory_v1",
+ credentials=credentials,
+ cache_discovery=False,
+ )
+ except Exception as e:
+ logger.warning("Could not create Google Directory client for group ACLs", error=str(e))
+ return []
+
+ roles: list[str] = []
+ seen: set[str] = set()
+ page_token: str | None = None
+ user_domain = user_email.rsplit("@", 1)[1].lower() if "@" in user_email else None
+
+ try:
+ while True:
+ kwargs: dict[str, Any] = {
+ "userKey": user_email,
+ "maxResults": 200,
+ "pageToken": page_token,
+ "fields": "nextPageToken,groups(email)",
+ }
+ if user_domain:
+ kwargs["domain"] = user_domain
+ request = directory_service.groups().list(
+ **kwargs,
+ )
+ response = await _execute_google_request(request)
+
+ for group in response.get("groups", []) or []:
+ role = google_drive_group_role(group.get("email"))
+ if role and role not in seen:
+ seen.add(role)
+ roles.append(role)
+
+ page_token = response.get("nextPageToken")
+ if not page_token:
+ break
+ except HttpError as e:
+ status = getattr(getattr(e, "resp", None), "status", None)
+ if status in (401, 403):
+ logger.warning(
+ "Google Directory group ACL lookup denied; check Admin SDK API and "
+ "admin.directory.group.readonly consent",
+ status_code=status,
+ )
+ return []
+ logger.warning("Google Directory group ACL lookup failed", error=str(e))
+ return []
+ except Exception as e:
+ logger.warning("Google Directory group ACL lookup errored", error=str(e))
+ return []
+
+ return roles
+
+
+async def get_current_user_google_principals(
+ drive_service: Any,
+ credentials: Any,
+) -> list[str]:
+ """Fetch canonical Google Drive user principals for the connected user."""
+ if credentials is None:
+ return []
+
+ user_email = await _get_drive_user_email(drive_service, credentials)
+ principal = google_drive_user_principal(user_email)
+ return [principal] if principal else []
+
+
+async def _get_cloud_identity_group_labels(
+ credentials: Any,
+ user_email: str,
+) -> list[dict[str, Any]] | None:
+ try:
+ cloud_identity_service = build(
+ "cloudidentity",
+ "v1",
+ credentials=credentials,
+ cache_discovery=False,
+ )
+ except Exception as e:
+ logger.debug("Could not create Google Cloud Identity client for group labels", error=str(e))
+ return None
+
+ labels: list[dict[str, Any]] = []
+ page_token: str | None = None
+ query = f"member_key_id == '{_cel_string(user_email)}' && '{GOOGLE_GROUP_LABEL}' in labels"
+
+ try:
+ while True:
+ request = (
+ cloud_identity_service.groups()
+ .memberships()
+ .searchTransitiveGroups(
+ parent="groups/-",
+ query=query,
+ pageSize=200,
+ pageToken=page_token,
+ )
+ )
+ response = await _execute_google_request(request)
+
+ for membership in response.get("memberships", []) or []:
+ group_key = membership.get("groupKey", {}) or {}
+ group_id = group_key.get("id")
+ principal = google_drive_group_role(group_id)
+ label = acl_principal_label(
+ principal,
+ kind="group",
+ provider=GOOGLE_DRIVE_GROUP_PROVIDER,
+ display_name=membership.get("displayName") or group_id,
+ email=group_id if isinstance(group_id, str) and "@" in group_id else None,
+ external_id=group_id,
+ )
+ if label:
+ labels.append(label)
+
+ page_token = response.get("nextPageToken")
+ if not page_token:
+ break
+ except HttpError as e:
+ status = getattr(getattr(e, "resp", None), "status", None)
+ if status in (401, 403):
+ return None
+ logger.warning("Google Cloud Identity group label lookup failed", error=str(e))
+ return None
+ except Exception as e:
+ logger.debug("Google Cloud Identity group label lookup errored", error=str(e))
+ return None
+
+ return unique_acl_principal_labels(labels)
+
+
+async def _get_directory_group_labels(credentials: Any, user_email: str) -> list[dict[str, Any]]:
+ try:
+ directory_service = build(
+ "admin",
+ "directory_v1",
+ credentials=credentials,
+ cache_discovery=False,
+ )
+ except Exception as e:
+ logger.warning("Could not create Google Directory client for group labels", error=str(e))
+ return []
+
+ labels: list[dict[str, Any]] = []
+ page_token: str | None = None
+ user_domain = user_email.rsplit("@", 1)[1].lower() if "@" in user_email else None
+
+ try:
+ while True:
+ kwargs: dict[str, Any] = {
+ "userKey": user_email,
+ "maxResults": 200,
+ "pageToken": page_token,
+ "fields": "nextPageToken,groups(email,name)",
+ }
+ if user_domain:
+ kwargs["domain"] = user_domain
+ response = await _execute_google_request(directory_service.groups().list(**kwargs))
+
+ for group in response.get("groups", []) or []:
+ email = group.get("email")
+ principal = google_drive_group_role(email)
+ label = acl_principal_label(
+ principal,
+ kind="group",
+ provider=GOOGLE_DRIVE_GROUP_PROVIDER,
+ display_name=group.get("name") or email,
+ email=email,
+ external_id=email,
+ )
+ if label:
+ labels.append(label)
+
+ page_token = response.get("nextPageToken")
+ if not page_token:
+ break
+ except HttpError as e:
+ status = getattr(getattr(e, "resp", None), "status", None)
+ if status in (401, 403):
+ logger.warning(
+ "Google Directory group label lookup denied; check Admin SDK API and consent",
+ status_code=status,
+ )
+ return []
+ logger.warning("Google Directory group label lookup failed", error=str(e))
+ return []
+ except Exception as e:
+ logger.warning("Google Directory group label lookup errored", error=str(e))
+ return []
+
+ return unique_acl_principal_labels(labels)
+
+
+async def get_current_user_google_principal_labels(
+ drive_service: Any,
+ credentials: Any,
+) -> list[dict[str, Any]]:
+ """Fetch display labels for Google Drive user/group principals."""
+ if credentials is None:
+ return []
+
+ user_email = await _get_drive_user_email(drive_service, credentials)
+ if not user_email:
+ return []
+
+ labels: list[dict[str, Any]] = []
+ user_principal = google_drive_user_principal(user_email)
+ user_label = acl_principal_label(
+ user_principal,
+ kind="user",
+ provider=GOOGLE_DRIVE_GROUP_PROVIDER,
+ display_name=user_email,
+ email=user_email,
+ external_id=user_email,
+ )
+ if user_label:
+ labels.append(user_label)
+
+ group_labels = await _get_cloud_identity_group_labels(credentials, user_email)
+ if group_labels is None:
+ group_labels = await _get_directory_group_labels(credentials, user_email)
+ labels.extend(group_labels)
+
+ return unique_acl_principal_labels(labels)
diff --git a/src/connectors/microsoft_graph_acl.py b/src/connectors/microsoft_graph_acl.py
new file mode 100644
index 000000000..1e2a17fb8
--- /dev/null
+++ b/src/connectors/microsoft_graph_acl.py
@@ -0,0 +1,395 @@
+"""Microsoft Graph group ACL helpers shared by Microsoft connectors."""
+
+from __future__ import annotations
+
+import inspect
+from typing import Any
+
+import httpx
+import jwt
+
+from utils.group_acl import (
+ acl_principal_label,
+ canonical_group_role,
+ canonical_group_roles,
+ canonical_user_principal,
+ unique_acl_principal_labels,
+)
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+MICROSOFT_GRAPH_GROUP_PROVIDER = "m365"
+
+
+def tenant_id_from_access_token(access_token: str | None, fallback: str | None = None) -> str:
+ """Read the tenant id from a Microsoft access token without validating it."""
+ if access_token:
+ raw_token = access_token.removeprefix("Bearer ").strip()
+ try:
+ claims = jwt.decode(
+ raw_token,
+ options={"verify_signature": False, "verify_aud": False},
+ )
+ token_tenant = claims.get("tid")
+ if token_tenant:
+ return token_tenant
+ except Exception as e:
+ logger.debug("Could not decode Microsoft access token tenant", error=str(e))
+ return fallback or "common"
+
+
+def microsoft_group_role(
+ group_id: str | None,
+ *,
+ access_token: str | None = None,
+ tenant_id: str | None = None,
+) -> str | None:
+ """Return the canonical OpenSearch role for a Microsoft group id."""
+ if not group_id:
+ return None
+ resolved_tenant = tenant_id_from_access_token(access_token, fallback=tenant_id)
+ return canonical_group_role(
+ MICROSOFT_GRAPH_GROUP_PROVIDER,
+ resolved_tenant,
+ group_id,
+ )
+
+
+def microsoft_user_principal(
+ user_identifier: str | None,
+ *,
+ access_token: str | None = None,
+ tenant_id: str | None = None,
+) -> str | None:
+ """Return the canonical DLS principal for a Microsoft Graph user identity."""
+ if not user_identifier:
+ return None
+ resolved_tenant = tenant_id_from_access_token(access_token, fallback=tenant_id)
+ return canonical_user_principal(
+ MICROSOFT_GRAPH_GROUP_PROVIDER,
+ resolved_tenant,
+ user_identifier,
+ )
+
+
+def microsoft_group_principal_label(
+ group_id: str | None,
+ *,
+ access_token: str | None = None,
+ tenant_id: str | None = None,
+ display_name: str | None = None,
+ email: str | None = None,
+) -> dict[str, Any] | None:
+ """Return non-authoritative display metadata for a Microsoft group principal."""
+ principal = microsoft_group_role(
+ group_id,
+ access_token=access_token,
+ tenant_id=tenant_id,
+ )
+ return acl_principal_label(
+ principal,
+ kind="group",
+ provider=MICROSOFT_GRAPH_GROUP_PROVIDER,
+ display_name=display_name or email or group_id,
+ email=email,
+ external_id=group_id,
+ )
+
+
+def microsoft_user_principal_label(
+ user_identifier: str | None,
+ *,
+ access_token: str | None = None,
+ tenant_id: str | None = None,
+ display_name: str | None = None,
+ email: str | None = None,
+ external_id: str | None = None,
+) -> dict[str, Any] | None:
+ """Return non-authoritative display metadata for a Microsoft user principal."""
+ principal = microsoft_user_principal(
+ user_identifier,
+ access_token=access_token,
+ tenant_id=tenant_id,
+ )
+ return acl_principal_label(
+ principal,
+ kind="user",
+ provider=MICROSOFT_GRAPH_GROUP_PROVIDER,
+ display_name=display_name or email or user_identifier,
+ email=email,
+ external_id=external_id or user_identifier,
+ )
+
+
+async def _maybe_await(value: Any) -> Any:
+ if inspect.isawaitable(value):
+ return await value
+ return value
+
+
+async def get_oauth_access_token(oauth: Any) -> str | None:
+ """Return an access token string from either old dict or current string APIs."""
+ if oauth is None:
+ return None
+ token_value = await _maybe_await(oauth.get_access_token())
+ if isinstance(token_value, dict):
+ return token_value.get("access_token")
+ if isinstance(token_value, str):
+ return token_value.removeprefix("Bearer ").strip()
+ return None
+
+
+async def get_current_user_microsoft_group_roles(
+ oauth: Any,
+ graph_base_url: str,
+ *,
+ tenant_id: str | None = None,
+ timeout_seconds: float = 10.0,
+) -> list[str]:
+ """Fetch transitive Microsoft group memberships for the current OAuth user."""
+ if oauth is None:
+ return []
+
+ try:
+ access_token = await get_oauth_access_token(oauth)
+ except Exception as e:
+ logger.warning("Unable to get Microsoft Graph token for group ACLs", error=str(e))
+ return []
+
+ if not access_token:
+ return []
+
+ resolved_tenant = tenant_id_from_access_token(access_token, fallback=tenant_id)
+ headers = {"Authorization": f"Bearer {access_token}"}
+ url = f"{graph_base_url}/me/transitiveMemberOf/microsoft.graph.group"
+ params: dict[str, str] | None = {"$select": "id"}
+ group_ids: list[str] = []
+
+ try:
+ async with httpx.AsyncClient(timeout=timeout_seconds) as client:
+ while url:
+ response = await client.get(url, headers=headers, params=params)
+ params = None
+ if response.status_code in (401, 403):
+ logger.warning(
+ "Microsoft Graph group ACL lookup denied",
+ status_code=response.status_code,
+ response_text=response.text[:500],
+ )
+ return []
+ if response.status_code != 200:
+ logger.warning(
+ "Microsoft Graph group ACL lookup failed",
+ status_code=response.status_code,
+ response_text=response.text[:500],
+ )
+ return []
+
+ data = response.json()
+ for entry in data.get("value", []):
+ group_id = entry.get("id")
+ if group_id:
+ group_ids.append(group_id)
+ url = data.get("@odata.nextLink")
+ except Exception as e:
+ logger.warning("Microsoft Graph group ACL lookup errored", error=str(e))
+ return []
+
+ return canonical_group_roles(
+ MICROSOFT_GRAPH_GROUP_PROVIDER,
+ resolved_tenant,
+ group_ids,
+ )
+
+
+def _decode_microsoft_user_identifiers(access_token: str, tenant_id: str | None) -> list[str]:
+ raw_token = access_token.removeprefix("Bearer ").strip()
+ try:
+ claims = jwt.decode(
+ raw_token,
+ options={"verify_signature": False, "verify_aud": False},
+ )
+ except Exception as e:
+ logger.debug("Could not decode Microsoft access token user identifiers", error=str(e))
+ return []
+
+ identifiers: list[str] = []
+ for claim in ("oid", "preferred_username", "upn", "email", "unique_name"):
+ value = claims.get(claim)
+ if value:
+ identifiers.append(str(value))
+
+ # If tenant_id was not in the access token, the principal helper will use
+ # this fallback. Returning identifiers here keeps JWT-only aliases useful
+ # even if /me is unavailable.
+ if tenant_id and claims.get("sub"):
+ identifiers.append(str(claims["sub"]))
+ return identifiers
+
+
+async def get_current_user_microsoft_principals(
+ oauth: Any,
+ graph_base_url: str,
+ *,
+ tenant_id: str | None = None,
+ timeout_seconds: float = 10.0,
+) -> list[str]:
+ """Fetch canonical Microsoft user principals for the current OAuth user."""
+ if oauth is None:
+ return []
+
+ try:
+ access_token = await get_oauth_access_token(oauth)
+ except Exception as e:
+ logger.warning("Unable to get Microsoft Graph token for user ACL aliases", error=str(e))
+ return []
+
+ if not access_token:
+ return []
+
+ identifiers = _decode_microsoft_user_identifiers(access_token, tenant_id)
+ resolved_tenant = tenant_id_from_access_token(access_token, fallback=tenant_id)
+
+ headers = {"Authorization": f"Bearer {access_token}"}
+ url = f"{graph_base_url}/me"
+ params = {"$select": "id,userPrincipalName,mail"}
+
+ try:
+ async with httpx.AsyncClient(timeout=timeout_seconds) as client:
+ response = await client.get(url, headers=headers, params=params)
+ if response.status_code == 200:
+ data = response.json()
+ for key in ("id", "userPrincipalName", "mail"):
+ value = data.get(key)
+ if value:
+ identifiers.append(str(value))
+ elif response.status_code in (401, 403):
+ logger.warning(
+ "Microsoft Graph user ACL alias lookup denied",
+ status_code=response.status_code,
+ response_text=response.text[:500],
+ )
+ else:
+ logger.warning(
+ "Microsoft Graph user ACL alias lookup failed",
+ status_code=response.status_code,
+ response_text=response.text[:500],
+ )
+ except Exception as e:
+ logger.warning("Microsoft Graph user ACL alias lookup errored", error=str(e))
+
+ principals: list[str] = []
+ seen: set[str] = set()
+ for identifier in identifiers:
+ principal = microsoft_user_principal(
+ identifier,
+ access_token=access_token,
+ tenant_id=resolved_tenant,
+ )
+ if principal and principal not in seen:
+ seen.add(principal)
+ principals.append(principal)
+ return principals
+
+
+async def get_current_user_microsoft_principal_labels(
+ oauth: Any,
+ graph_base_url: str,
+ *,
+ tenant_id: str | None = None,
+ timeout_seconds: float = 10.0,
+) -> list[dict[str, Any]]:
+ """Fetch display labels for current Microsoft user and group principals."""
+ if oauth is None:
+ return []
+
+ try:
+ access_token = await get_oauth_access_token(oauth)
+ except Exception as e:
+ logger.warning("Unable to get Microsoft Graph token for principal labels", error=str(e))
+ return []
+
+ if not access_token:
+ return []
+
+ resolved_tenant = tenant_id_from_access_token(access_token, fallback=tenant_id)
+ identifiers = _decode_microsoft_user_identifiers(access_token, tenant_id)
+ labels: list[dict[str, Any]] = []
+ headers = {"Authorization": f"Bearer {access_token}"}
+
+ try:
+ async with httpx.AsyncClient(timeout=timeout_seconds) as client:
+ response = await client.get(
+ f"{graph_base_url}/me",
+ headers=headers,
+ params={"$select": "id,displayName,userPrincipalName,mail"},
+ )
+ if response.status_code == 200:
+ data = response.json()
+ user_email = data.get("mail") or data.get("userPrincipalName")
+ for key in ("id", "userPrincipalName", "mail"):
+ value = data.get(key)
+ if value:
+ identifiers.append(str(value))
+ for identifier in identifiers:
+ label = microsoft_user_principal_label(
+ identifier,
+ access_token=access_token,
+ tenant_id=resolved_tenant,
+ display_name=data.get("displayName") or user_email,
+ email=user_email,
+ external_id=identifier,
+ )
+ if label:
+ labels.append(label)
+ elif response.status_code in (401, 403):
+ logger.warning(
+ "Microsoft Graph user principal label lookup denied",
+ status_code=response.status_code,
+ response_text=response.text[:500],
+ )
+ else:
+ logger.warning(
+ "Microsoft Graph user principal label lookup failed",
+ status_code=response.status_code,
+ response_text=response.text[:500],
+ )
+
+ group_url = f"{graph_base_url}/me/transitiveMemberOf/microsoft.graph.group"
+ params: dict[str, str] | None = {"$select": "id,displayName,mail"}
+ while group_url:
+ response = await client.get(group_url, headers=headers, params=params)
+ params = None
+ if response.status_code in (401, 403):
+ logger.warning(
+ "Microsoft Graph group principal label lookup denied",
+ status_code=response.status_code,
+ response_text=response.text[:500],
+ )
+ break
+ if response.status_code != 200:
+ logger.warning(
+ "Microsoft Graph group principal label lookup failed",
+ status_code=response.status_code,
+ response_text=response.text[:500],
+ )
+ break
+
+ data = response.json()
+ for entry in data.get("value", []):
+ group_id = entry.get("id")
+ label = microsoft_group_principal_label(
+ group_id,
+ access_token=access_token,
+ tenant_id=resolved_tenant,
+ display_name=entry.get("displayName") or entry.get("mail"),
+ email=entry.get("mail"),
+ )
+ if label:
+ labels.append(label)
+ group_url = data.get("@odata.nextLink")
+ except Exception as e:
+ logger.warning("Microsoft Graph principal label lookup errored", error=str(e))
+
+ return unique_acl_principal_labels(labels)
diff --git a/src/connectors/onedrive/connector.py b/src/connectors/onedrive/connector.py
index 3f0b9cc38..82652b0cf 100644
--- a/src/connectors/onedrive/connector.py
+++ b/src/connectors/onedrive/connector.py
@@ -6,7 +6,19 @@ from urllib.parse import urlparse
import httpx
+from utils.group_acl import unique_acl_principal_labels
+
from ..base import BaseConnector, ConnectorDocument, DocumentACL
+from ..microsoft_graph_acl import (
+ get_current_user_microsoft_group_roles,
+ get_current_user_microsoft_principal_labels,
+ get_current_user_microsoft_principals,
+ get_oauth_access_token,
+ microsoft_group_principal_label,
+ microsoft_group_role,
+ microsoft_user_principal,
+ microsoft_user_principal_label,
+)
from .oauth import OneDriveOAuth
logger = logging.getLogger(__name__)
@@ -97,7 +109,9 @@ class OneDriveConnector(BaseConnector):
# Graph API defaults
self._graph_api_version = "v1.0"
- self._default_params: dict[str, Any] = {}
+ self._default_params: dict[str, Any] = {
+ "$select": "id,name,size,lastModifiedDateTime,createdDateTime,webUrl,file,folder,@microsoft.graph.downloadUrl"
+ }
# Selective sync support (similar to Google Drive)
self.cfg = type(
@@ -132,6 +146,27 @@ class OneDriveConnector(BaseConnector):
"""Set base URL"""
self._base_url = value
+ async def get_current_user_group_roles(self) -> list[str]:
+ """Return canonical group ACL roles for the connected Microsoft user."""
+ return await get_current_user_microsoft_group_roles(
+ self.oauth,
+ self._graph_base_url,
+ )
+
+ async def get_current_user_principals(self) -> list[str]:
+ """Return canonical user ACL principals for the connected Microsoft user."""
+ return await get_current_user_microsoft_principals(
+ self.oauth,
+ self._graph_base_url,
+ )
+
+ async def get_current_user_principal_labels(self) -> list[dict[str, Any]]:
+ """Return display labels for current Microsoft user/group ACL principals."""
+ return await get_current_user_microsoft_principal_labels(
+ self.oauth,
+ self._graph_base_url,
+ )
+
def set_file_infos(self, file_infos: list[dict[str, Any]]) -> None:
"""
Cache file metadata including download URLs for later use.
@@ -427,7 +462,7 @@ class OneDriveConnector(BaseConnector):
"""
try:
# Get access token
- access_token = self.oauth.get_access_token()
+ access_token = await get_oauth_access_token(self.oauth)
if not access_token:
logger.warning(f"No access token available for ACL extraction: {file_id}")
@@ -450,23 +485,66 @@ class OneDriveConnector(BaseConnector):
allowed_users = []
allowed_groups = []
+ allowed_principals = []
+ allowed_principal_labels = []
owner = None
for perm in permissions_data.get("value", []):
roles = perm.get("roles", []) # ["read", "write", "owner"]
- # Granted to user
- if "grantedTo" in perm:
- user_info = perm["grantedTo"].get("user", {})
+ # Granted to user/group (grantedToV2 is the current Graph shape;
+ # grantedTo is retained for older responses).
+ granted_to = perm.get("grantedToV2") or perm.get("grantedTo")
+ if granted_to:
+ user_info = granted_to.get("user", {})
email = user_info.get("email")
if email:
allowed_users.append(email)
if "owner" in roles:
owner = email
+ for identifier in (
+ user_info.get("id"),
+ user_info.get("userPrincipalName"),
+ email,
+ ):
+ user_principal = microsoft_user_principal(
+ identifier,
+ access_token=access_token,
+ )
+ if user_principal:
+ allowed_principals.append(user_principal)
+ label = microsoft_user_principal_label(
+ identifier,
+ access_token=access_token,
+ display_name=user_info.get("displayName") or email,
+ email=email,
+ external_id=identifier,
+ )
+ if label:
+ allowed_principal_labels.append(label)
+ group_info = granted_to.get("group", {})
+ group_role = microsoft_group_role(
+ group_info.get("id"),
+ access_token=access_token,
+ )
+ if group_role:
+ allowed_groups.append(group_role)
+ allowed_principals.append(group_role)
+ label = microsoft_group_principal_label(
+ group_info.get("id"),
+ access_token=access_token,
+ display_name=group_info.get("displayName") or group_info.get("email"),
+ email=group_info.get("email"),
+ )
+ if label:
+ allowed_principal_labels.append(label)
# Granted to identities (can include users and groups)
- elif "grantedToIdentities" in perm:
- for identity in perm["grantedToIdentities"]:
+ identities = (
+ perm.get("grantedToIdentitiesV2") or perm.get("grantedToIdentities") or []
+ )
+ if identities:
+ for identity in identities:
# User
if "user" in identity:
user_info = identity["user"]
@@ -475,19 +553,45 @@ class OneDriveConnector(BaseConnector):
allowed_users.append(email)
if "owner" in roles:
owner = email
+ for identifier in (
+ user_info.get("id"),
+ user_info.get("userPrincipalName"),
+ email,
+ ):
+ user_principal = microsoft_user_principal(
+ identifier,
+ access_token=access_token,
+ )
+ if user_principal:
+ allowed_principals.append(user_principal)
# Group
- elif "group" in identity:
+ if "group" in identity:
group_info = identity["group"]
group_id = group_info.get("id")
- group_display_name = group_info.get("displayName", group_id)
- if group_id:
- allowed_groups.append(group_display_name)
+ group_role = microsoft_group_role(
+ group_id,
+ access_token=access_token,
+ )
+ if group_role:
+ allowed_groups.append(group_role)
+ allowed_principals.append(group_role)
+ label = microsoft_group_principal_label(
+ group_id,
+ access_token=access_token,
+ display_name=group_info.get("displayName")
+ or group_info.get("email"),
+ email=group_info.get("email"),
+ )
+ if label:
+ allowed_principal_labels.append(label)
return DocumentACL(
owner=owner,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
+ allowed_principals=allowed_principals,
+ allowed_principal_labels=unique_acl_principal_labels(allowed_principal_labels),
)
except Exception as e:
@@ -507,9 +611,7 @@ class OneDriveConnector(BaseConnector):
logger.info(f"Using cached download URL for file {file_id}")
content = await self._download_file_from_url(cached_info["downloadUrl"])
- acl = DocumentACL(
- owner="",
- )
+ acl = DocumentACL(owner="")
return ConnectorDocument(
id=file_id,
@@ -672,10 +774,10 @@ class OneDriveConnector(BaseConnector):
except Exception as e:
logger.debug(f"Shares approach {i + 1} failed: {e}")
- # Try: /drives/{driveId}/items/{file_id} with full file ID
- logger.info(f"Trying drives endpoint: /drives/{drive_id}/items/{file_id}")
+ # Try: /drives/{driveId}/items/{itemId} with full item ID (including 's' prefix)
+ logger.info(f"Trying drives endpoint: /drives/{drive_id}/items/{item_id}")
try:
- url = f"{self._graph_base_url}/drives/{drive_id}/items/{file_id}"
+ url = f"{self._graph_base_url}/drives/{drive_id}/items/{item_id}"
response = await self._make_graph_request(url, params=params)
if response.status_code == 200:
return response.json()
@@ -684,20 +786,23 @@ class OneDriveConnector(BaseConnector):
f"Drives endpoint failed with status {response.status_code}: {response.text}"
)
except Exception as e:
- logger.debug(f"Drives endpoint exception: {e}")
+ logger.debug(f"Drives endpoint failed: {e}")
- # Try: /drives/{driveId}/items/{clean_file_id} without 's' prefix
+ # Try: /drives/{driveId}/items/{itemId} without 's' prefix
if item_id.startswith("s"):
clean_item_id = item_id[1:] # Remove 's' prefix
- clean_file_id = f"{drive_id}!{clean_item_id}"
logger.info(
- f"Trying drives endpoint without 's' prefix: /drives/{drive_id}/items/{clean_file_id}"
+ f"Trying drives endpoint without 's' prefix: /drives/{drive_id}/items/{clean_item_id}"
)
try:
- url = f"{self._graph_base_url}/drives/{drive_id}/items/{clean_file_id}"
+ url = f"{self._graph_base_url}/drives/{drive_id}/items/{clean_item_id}"
response = await self._make_graph_request(url, params=params)
if response.status_code == 200:
return response.json()
+ else:
+ logger.warning(
+ f"Drives endpoint without 's' prefix failed with status {response.status_code}: {response.text}"
+ )
except Exception as e:
logger.debug(f"Drives endpoint (no prefix) failed: {e}")
diff --git a/src/connectors/service.py b/src/connectors/service.py
index 838176738..84c36a3ee 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -45,6 +45,37 @@ class ConnectorService:
"""Get a connector by connection ID"""
return await self.connection_manager.get_connector(connection_id)
+ async def _get_effective_sync_jwt(
+ self,
+ user_id: str,
+ jwt_token: str | None = None,
+ ) -> str | None:
+ """Return a current OpenSearch JWT for connector sync work."""
+ if not self.session_manager:
+ return jwt_token
+
+ user = self.session_manager.get_user(user_id)
+ if user is None and user_id:
+ from session_manager import User
+
+ user = User(
+ user_id=user_id,
+ email=user_id,
+ name=user_id,
+ provider="connector",
+ )
+ self.session_manager.users[user_id] = user
+ if user is None:
+ return self.session_manager.get_effective_jwt_token(user_id, jwt_token)
+
+ effective_token = jwt_token or user.jwt_token
+ if (
+ effective_token is None
+ and getattr(self.session_manager, "private_key", None) is not None
+ ):
+ return self.session_manager.create_jwt_token(user)
+ return self.session_manager.get_effective_jwt_token(user.user_id, effective_token)
+
async def process_connector_document(
self,
document: ConnectorDocument,
@@ -56,6 +87,7 @@ class ConnectorService:
ingest_settings: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Process a document from a connector using active processing pipeline"""
+ jwt_token = await self._get_effective_sync_jwt(owner_user_id, jwt_token)
from config.settings import DISABLE_INGEST_WITH_LANGFLOW
@@ -70,10 +102,14 @@ class ConnectorService:
allowed_users = []
allowed_groups = []
+ allowed_principals = []
+ allowed_principal_labels = []
if document.acl:
try:
allowed_users = document.acl.allowed_users or []
allowed_groups = document.acl.allowed_groups or []
+ allowed_principals = document.acl.allowed_principals or []
+ allowed_principal_labels = document.acl.allowed_principal_labels or []
except AttributeError:
pass
@@ -103,6 +139,8 @@ class ConnectorService:
source_url=document.source_url,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
+ allowed_principals=allowed_principals,
+ allowed_principal_labels=allowed_principal_labels,
)
return {
"status": "indexed",
@@ -180,6 +218,7 @@ class ConnectorService:
owner_user_id: str,
connector_type: str,
jwt_token: str = None,
+ id_field: str = "document_id",
):
"""Update indexed chunks with connector-specific metadata"""
from utils.acl_utils import update_document_acl
@@ -190,6 +229,9 @@ class ConnectorService:
opensearch_client = self.session_manager.get_user_opensearch_client(
owner_user_id, jwt_token
)
+ write_client = self.clients.opensearch
+ if write_client is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
# Update ACL if changed (hash-based skip optimization).
# Match both document_id and connector_file_id: non-Langflow connector
@@ -199,6 +241,7 @@ class ConnectorService:
document_id=document.id,
acl=document.acl,
opensearch_client=opensearch_client,
+ write_opensearch_client=write_client,
id_fields=("document_id", "connector_file_id"),
)
@@ -213,9 +256,10 @@ class ConnectorService:
logger.error(f"ACL update error for {document.id}: {acl_result.get('error')}")
# Update other metadata fields (source_url, timestamps, etc.)
- # Use update_by_query for efficiency
+ # Use the backend client for writes; the scoped client above is only
+ # used for DLS visibility/ACL-change checks.
try:
- await opensearch_client.update_by_query(
+ await write_client.update_by_query(
index=self.index_name,
body={
# Match both fields: non-Langflow chunks carry the connector id
@@ -292,6 +336,8 @@ class ConnectorService:
in this set will be synced. Used to prevent deleted files
from being re-synced.
"""
+ jwt_token = await self._get_effective_sync_jwt(user_id, jwt_token)
+
if not self.task_service:
raise ValueError(
"TaskService not available - connector sync requires task service dependency"
@@ -418,6 +464,8 @@ class ConnectorService:
ingest_settings: Optional UI-style dict (``embeddingModel``, ``chunkSize``, …) passed to
``ConnectorFileProcessor`` when Langflow ingest is disabled.
"""
+ jwt_token = await self._get_effective_sync_jwt(user_id, jwt_token)
+
if not self.task_service:
raise ValueError(
"TaskService not available - connector sync requires task service dependency"
diff --git a/src/connectors/sharepoint/connector.py b/src/connectors/sharepoint/connector.py
index 825b83884..0a2a78f86 100644
--- a/src/connectors/sharepoint/connector.py
+++ b/src/connectors/sharepoint/connector.py
@@ -5,9 +5,20 @@ from urllib.parse import urlparse
import httpx
+from utils.group_acl import unique_acl_principal_labels
from utils.logging_config import get_logger
from ..base import BaseConnector, ConnectorDocument, DocumentACL
+from ..microsoft_graph_acl import (
+ get_current_user_microsoft_group_roles,
+ get_current_user_microsoft_principal_labels,
+ get_current_user_microsoft_principals,
+ get_oauth_access_token,
+ microsoft_group_principal_label,
+ microsoft_group_role,
+ microsoft_user_principal,
+ microsoft_user_principal_label,
+)
from .oauth import SharePointOAuth
logger = get_logger(__name__)
@@ -145,6 +156,30 @@ class SharePointConnector(BaseConnector):
"""Set base URL (updates sharepoint_url internally)"""
self.sharepoint_url = value
+ async def get_current_user_group_roles(self) -> list[str]:
+ """Return canonical group ACL roles for the connected Microsoft user."""
+ return await get_current_user_microsoft_group_roles(
+ self.oauth,
+ self._graph_base_url,
+ tenant_id=self.tenant_id,
+ )
+
+ async def get_current_user_principals(self) -> list[str]:
+ """Return canonical user ACL principals for the connected Microsoft user."""
+ return await get_current_user_microsoft_principals(
+ self.oauth,
+ self._graph_base_url,
+ tenant_id=self.tenant_id,
+ )
+
+ async def get_current_user_principal_labels(self) -> list[dict[str, Any]]:
+ """Return display labels for current Microsoft user/group ACL principals."""
+ return await get_current_user_microsoft_principal_labels(
+ self.oauth,
+ self._graph_base_url,
+ tenant_id=self.tenant_id,
+ )
+
def set_file_infos(self, file_infos: list[dict[str, Any]]) -> None:
"""
Cache file metadata including download URLs for later use.
@@ -488,7 +523,7 @@ class SharePointConnector(BaseConnector):
"""
try:
# Get access token - use same approach as _make_graph_request
- access_token = self.oauth.get_access_token()
+ access_token = await get_oauth_access_token(self.oauth)
if not access_token:
logger.warning(f"No access token available for ACL extraction: {file_id}")
@@ -517,6 +552,8 @@ class SharePointConnector(BaseConnector):
allowed_users = []
allowed_groups = []
+ allowed_principals = []
+ allowed_principal_labels = []
owner = None
for perm in permissions_data.get("value", []):
@@ -533,6 +570,46 @@ class SharePointConnector(BaseConnector):
allowed_users.append(user_identifier)
if "owner" in roles:
owner = user_identifier
+ for identifier in (
+ user_info.get("id"),
+ user_info.get("userPrincipalName"),
+ email,
+ ):
+ user_principal = microsoft_user_principal(
+ identifier,
+ access_token=access_token,
+ tenant_id=self.tenant_id,
+ )
+ if user_principal:
+ allowed_principals.append(user_principal)
+ label = microsoft_user_principal_label(
+ identifier,
+ access_token=access_token,
+ tenant_id=self.tenant_id,
+ display_name=display_name or email,
+ email=email,
+ external_id=identifier,
+ )
+ if label:
+ allowed_principal_labels.append(label)
+ group_info = granted_to.get("group", {})
+ group_role = microsoft_group_role(
+ group_info.get("id"),
+ access_token=access_token,
+ tenant_id=self.tenant_id,
+ )
+ if group_role:
+ allowed_groups.append(group_role)
+ allowed_principals.append(group_role)
+ label = microsoft_group_principal_label(
+ group_info.get("id"),
+ access_token=access_token,
+ tenant_id=self.tenant_id,
+ display_name=group_info.get("displayName") or group_info.get("email"),
+ email=group_info.get("email"),
+ )
+ if label:
+ allowed_principal_labels.append(label)
# Granted to identities (can include users and groups)
if "grantedToIdentitiesV2" in perm or "grantedToIdentities" in perm:
@@ -550,19 +627,48 @@ class SharePointConnector(BaseConnector):
allowed_users.append(user_identifier)
if "owner" in roles:
owner = user_identifier
+ for identifier in (
+ user_info.get("id"),
+ user_info.get("userPrincipalName"),
+ email,
+ ):
+ user_principal = microsoft_user_principal(
+ identifier,
+ access_token=access_token,
+ tenant_id=self.tenant_id,
+ )
+ if user_principal:
+ allowed_principals.append(user_principal)
# Group
if "group" in identity:
group_info = identity["group"]
group_id = group_info.get("id")
- group_display_name = group_info.get("displayName", group_id)
- if group_id or group_display_name:
- allowed_groups.append(group_display_name or group_id)
+ group_role = microsoft_group_role(
+ group_id,
+ access_token=access_token,
+ tenant_id=self.tenant_id,
+ )
+ if group_role:
+ allowed_groups.append(group_role)
+ allowed_principals.append(group_role)
+ label = microsoft_group_principal_label(
+ group_id,
+ access_token=access_token,
+ tenant_id=self.tenant_id,
+ display_name=group_info.get("displayName")
+ or group_info.get("email"),
+ email=group_info.get("email"),
+ )
+ if label:
+ allowed_principal_labels.append(label)
return DocumentACL(
owner=owner,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
+ allowed_principals=allowed_principals,
+ allowed_principal_labels=unique_acl_principal_labels(allowed_principal_labels),
)
except Exception as e:
diff --git a/src/dependencies.py b/src/dependencies.py
index c3de68bda..36b77b123 100644
--- a/src/dependencies.py
+++ b/src/dependencies.py
@@ -113,10 +113,26 @@ def get_connector_service(services: dict = Depends(get_services)):
return services["connector_service"]
+def get_group_acl_service(services: dict = Depends(get_services)):
+ return services.get("group_acl_service")
+
+
+def get_dls_principal_service(services: dict = Depends(get_services)):
+ return services.get("dls_principal_service")
+
+
def get_langflow_file_service(services: dict = Depends(get_services)):
return services["langflow_file_service"]
+def get_document_index_writer(services: dict = Depends(get_services)):
+ return services["document_index_writer"]
+
+
+def get_langflow_ingest_token_service(services: dict = Depends(get_services)):
+ return services["langflow_ingest_token_service"]
+
+
def get_models_service(services: dict = Depends(get_services)):
return services["models_service"]
@@ -259,6 +275,58 @@ async def _attach_db_user_id(request: Request, user: User | None) -> User | None
return user_with_db_id
+async def _attach_opensearch_jwt(
+ request: Request,
+ user: User | None,
+ session_manager,
+ token_hint: str | None = None,
+) -> User | None:
+ """Attach OpenSearch auth state and refresh DLS principal lookup state."""
+ if user is None:
+ return None
+
+ effective_token = session_manager.get_effective_jwt_token(
+ user.user_id,
+ token_hint if token_hint is not None else user.jwt_token,
+ )
+ if effective_token != user.jwt_token:
+ user = dataclasses.replace(user, jwt_token=effective_token)
+
+ services = getattr(getattr(request, "app", None), "state", None)
+ services = getattr(services, "services", {}) or {}
+ dls_principal_service = services.get("dls_principal_service")
+
+ if dls_principal_service is not None:
+ try:
+ principals = await dls_principal_service.refresh_user_principals(user)
+ request.state.opensearch_dls_principals = principals
+ except Exception as e:
+ logger.warning(
+ "Failed to refresh OpenSearch DLS principals",
+ user_id=user.user_id,
+ error=str(e),
+ )
+
+ request.state.opensearch_group_roles = []
+
+ return user
+
+
+async def _attach_request_user(
+ request: Request,
+ user: User | None,
+ session_manager,
+ token_hint: str | None = None,
+) -> User | None:
+ user_with_opensearch_jwt = await _attach_opensearch_jwt(
+ request,
+ user,
+ session_manager,
+ token_hint=token_hint,
+ )
+ return await _attach_db_user_id(request, user_with_opensearch_jwt)
+
+
def invalidate_user_ensured_cache(
oauth_provider: str | None = None,
oauth_subject: str | None = None,
@@ -392,7 +460,7 @@ def require_all_permissions(required_perms: Sequence[str]):
# ─────────────────────────────────────────────
-# IBM AMS authentication helper
+# Upstream authentication helper
# ─────────────────────────────────────────────
@@ -425,18 +493,13 @@ def _stage_jwt_roles(request: Request, claims: dict, user_id: str | None) -> Non
async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:
- """Authenticate via IBM AMS.
-
- 0. X-IBM-LH-Credentials header (configurable via IBM_CREDENTIALS_HEADER) —
- injected by Traefik on every forwarded request. Contains Basic credentials
- for OpenSearch. Decoded and persisted to connections.json per user.
- 1. ibm-openrag-session cookie — set by Traefik after validating credentials
- with AMS. JWT is decoded without re-validation (Traefik already validated).
- When RBAC/JWT-role sync is enabled, this JWT is instead read from the
- gateway-forwarded header named by ``get_jwt_auth_header()`` (the cookie is
- used only when RBAC is off); identity and roles both come from that token.
- 2. ibm-auth-basic cookie — local dev fallback set by our ibm_login endpoint
- when Traefik is not present.
+ """Authenticate via upstream auth.
+
+ 0. Configured credentials header containing OpenSearch credentials.
+ 1. Configured session cookie. When JWT-role sync is enabled, the JWT is
+ instead read from the gateway-forwarded header named by ``get_jwt_auth_header()``;
+ identity and roles both come from that token.
+ 2. Local dev basic-auth cookie.
If *required* is True, raises HTTP 401 when none is present.
If *required* is False, returns None instead of raising.
@@ -516,6 +579,9 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:
logger.debug("[AUTH] IBM LH credentials found in request headers")
opensearch_username, _ = extract_ibm_credentials(lh_credentials)
logger.debug("[AUTH] IBM LH credentials extracted successfully")
+ user_id = user_id or opensearch_username
+ email = email or opensearch_username
+ name = name or opensearch_username
# Persist credentials to connections.json for reuse by background processes
connector_service = request.app.state.services.get("connector_service")
@@ -542,8 +608,6 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:
request.state.user = user
return user
- # ── Option 1: ibm-openrag-session cookie (production via Traefik) ───
- # ibm_token = request.cookies.get(IBM_SESSION_COOKIE_NAME)
if ibm_token and user_id:
logger.debug("[AUTH] IBM JWT cookie present and user_id found")
logger.debug("[AUTH] LH credentials not available in header, reading from connections.json")
@@ -588,7 +652,6 @@ async def _get_ibm_user(request: Request, required: bool) -> Optional["User"]:
request.state.user = None
return None
- # ── Option 2: ibm-auth-basic cookie (local dev, no Traefik) ─────────
auth_header = request.cookies.get("ibm-auth-basic", "")
if auth_header.startswith("Basic "):
logger.debug("[AUTH] Debug mode enabled, extracting IBM LH credentials from cookie")
@@ -633,19 +696,17 @@ async def get_current_user(
from config.settings import IBM_AUTH_ENABLED, is_no_auth_mode
from session_manager import AnonymousUser
- # IBM AMS cookie auth takes priority when enabled
+ # Upstream cookie auth takes priority when enabled.
if IBM_AUTH_ENABLED:
logger.debug("[AUTH] IBM auth mode enabled, getting current user")
user = await _get_ibm_user(request, required=True)
if user and user.user_id and user.user_id not in session_manager.users:
session_manager.users[user.user_id] = user
- return await _attach_db_user_id(request, user)
+ return await _attach_request_user(request, user, session_manager)
if is_no_auth_mode():
user = AnonymousUser()
- effective_token = session_manager.get_effective_jwt_token(None, None)
- user_with_token = dataclasses.replace(user, jwt_token=effective_token)
- return await _attach_db_user_id(request, user_with_token)
+ return await _attach_request_user(request, user, session_manager)
auth_token = request.cookies.get("auth_token")
if not auth_token:
@@ -655,11 +716,12 @@ async def get_current_user(
if not user:
raise HTTPException(status_code=401, detail="Authentication required")
- # get_effective_jwt_token handles anonymous JWT creation if needed
- effective_token = session_manager.get_effective_jwt_token(user.user_id, auth_token)
- user_with_token = dataclasses.replace(user, jwt_token=effective_token)
-
- return await _attach_db_user_id(request, user_with_token)
+ return await _attach_request_user(
+ request,
+ user,
+ session_manager,
+ token_hint=auth_token,
+ )
async def get_optional_user(
@@ -675,22 +737,20 @@ async def get_optional_user(
from config.settings import IBM_AUTH_ENABLED, is_no_auth_mode
from session_manager import AnonymousUser
- # IBM AMS cookie auth takes priority when enabled
+ # Upstream cookie auth takes priority when enabled.
if IBM_AUTH_ENABLED:
logger.debug("[AUTH] IBM auth mode enabled, getting optional user")
user = await _get_ibm_user(request, required=False)
if user and user.user_id and user.user_id not in session_manager.users:
session_manager.users[user.user_id] = user
if user:
- return await _attach_db_user_id(request, user)
+ return await _attach_request_user(request, user, session_manager)
request.state.db_user_id = None
return None
if is_no_auth_mode():
user = AnonymousUser()
- effective_token = session_manager.get_effective_jwt_token(None, None)
- user_with_token = dataclasses.replace(user, jwt_token=effective_token)
- return await _attach_db_user_id(request, user_with_token)
+ return await _attach_request_user(request, user, session_manager)
auth_token = request.cookies.get("auth_token")
if not auth_token:
@@ -698,14 +758,13 @@ async def get_optional_user(
return None
user = session_manager.get_user_from_token(auth_token)
- # get_effective_jwt_token handles anonymous JWT creation if needed
- effective_token = (
- session_manager.get_effective_jwt_token(user.user_id, auth_token) if user else None
- )
- user_with_token = dataclasses.replace(user, jwt_token=effective_token) if user else None
-
- if user_with_token:
- return await _attach_db_user_id(request, user_with_token)
+ if user:
+ return await _attach_request_user(
+ request,
+ user,
+ session_manager,
+ token_hint=auth_token,
+ )
request.state.user = None
request.state.db_user_id = None
return None
@@ -717,7 +776,7 @@ async def get_api_key_user_async(
session_manager=Depends(get_session_manager),
) -> User:
"""
- Async dependency: require API key or IBM authentication.
+ Async dependency: require API key or upstream authentication.
Accepts:
- A gateway-forwarded JWT in the configurable OPENRAG_JWT_AUTH_HEADER
@@ -725,7 +784,7 @@ async def get_api_key_user_async(
source of identity; under RBAC it also supplies (and enforces) roles.
- X-API-Key: orag_... header
- Authorization: Bearer orag_... header
- - X-Username + X-Api-Key headers (when IBM_AUTH_ENABLED)
+ - X-Username + X-Api-Key headers when upstream auth mode is enabled
Raises HTTP 401 if no valid credentials are provided.
"""
@@ -767,10 +826,9 @@ async def get_api_key_user_async(
raise HTTPException(status_code=401, detail="Invalid or unverifiable JWT")
# RBAC off + missing/invalid JWT -> fall through to the API-key path.
- # IBM auth path: X-Username + X-Api-Key forwarded by the MCP via the SDK
from config.settings import IBM_AUTH_ENABLED
- # IBM auth path: X-Username + X-Api-Key forwarded by the MCP via the SDK
+ # Upstream auth path: X-Username + X-Api-Key forwarded by the MCP via the SDK.
if IBM_AUTH_ENABLED:
ibm_username = request.headers.get("X-Username")
ibm_api_key = request.headers.get("X-Api-Key")
@@ -789,8 +847,7 @@ async def get_api_key_user_async(
opensearch_username=ibm_username,
opensearch_credentials=ibm_api_key_b64,
)
- request.state.user = user
- return await _attach_db_user_id(request, user)
+ return await _attach_request_user(request, user, session_manager)
# API key path
api_key = request.headers.get("X-API-Key")
@@ -832,13 +889,10 @@ async def get_api_key_user_async(
if user.user_id not in session_manager.users:
session_manager.users[user.user_id] = user
- effective_token = session_manager.get_effective_jwt_token(user.user_id, None)
- user_with_token = dataclasses.replace(user, jwt_token=effective_token)
-
request.state.api_key_id = user_info["key_id"]
# Phase 2 will populate api_key_role_ids from the SQL api_keys table.
# In Phase 1 we leave it unset so require_permission falls back to the
# user's live role membership (no privilege escalation possible).
request.state.api_key_role_ids = getattr(request.state, "api_key_role_ids", None)
- return await _attach_db_user_id(request, user_with_token)
+ return await _attach_request_user(request, user, session_manager)
diff --git a/src/models/processors.py b/src/models/processors.py
index 4174cca62..2745d679b 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -1,5 +1,4 @@
import asyncio
-import datetime
import mimetypes
import os
import time
@@ -11,7 +10,6 @@ from utils.document_processing import (
process_text_file,
resplit_chunks_character_windows,
)
-from utils.embedding_fields import ensure_embedding_field_exists
from utils.file_utils import (
auto_cleanup_tempfile,
clean_connector_filename,
@@ -21,7 +19,7 @@ from utils.file_utils import (
)
from utils.hash_utils import hash_id
from utils.logging_config import get_logger
-from utils.opensearch_queries import build_filename_delete_body, build_filename_search_body
+from utils.opensearch_queries import build_filename_search_body
from .tasks import FileTask, TaskStatus, UploadTask
@@ -160,25 +158,46 @@ class TaskProcessor:
self,
filename: str,
opensearch_client,
+ owner_user_id: str | None = None,
) -> None:
"""
Delete all chunks of a document with the given filename from OpenSearch.
"""
+ from config.settings import clients, get_index_name
+ from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
+ from utils.opensearch_queries import build_owned_filename_query
+
try:
+ write_client = clients.opensearch
+ if write_client is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
+
deleted_count = 0
+ if not owner_user_id:
+ logger.warning(
+ "Skipped delete_by_filename because owner_user_id is missing",
+ filename=filename,
+ )
+ return
+
candidate_filenames = get_filename_aliases(filename)
if not candidate_filenames:
logger.info(
- "Skipped delete_by_filename due to empty filename input",
+ "Skipped delete_by_filename because filename input is empty",
filename=filename,
)
return
for candidate in candidate_filenames:
- delete_body = build_filename_delete_body(candidate)
- response = await opensearch_client.delete_by_query(
- index=get_index_name(), body=delete_body
+ document_ids = await collect_visible_document_ids(
+ opensearch_client,
+ index=get_index_name(),
+ query=build_owned_filename_query(candidate, owner_user_id),
+ )
+ deleted_count += await delete_document_ids(
+ write_client,
+ index=get_index_name(),
+ document_ids=document_ids,
)
- deleted_count += response.get("deleted", 0)
logger.info(
"Deleted existing document chunks", filename=filename, deleted_count=deleted_count
)
@@ -311,32 +330,39 @@ class TaskProcessor:
)
return {"status": "error", "error": "No text content could be extracted from document"}
- dimensions = len(embeddings[0])
-
- # Ensure the embedding field exists for this model
- embedding_field_name = await ensure_embedding_field_exists(
- opensearch_client, embedding_model, get_index_name(), dimensions
+ from services.document_index_writer import (
+ DocumentIndexChunk,
+ DocumentIndexContext,
+ DocumentIndexWriter,
)
+ document_index_writer = getattr(self.document_service, "document_index_writer", None)
+ if document_index_writer is None:
+ document_index_writer = DocumentIndexWriter()
+
# Clear stale chunks from a prior indexing of this document. Chunks are
- # stored under ids {file_hash}_{i}; if the new chunk count is lower than
- # the prior one, trailing chunks (e.g. with an old filename after a
- # SharePoint rename) would otherwise survive the per-chunk upsert.
- # DLS-safe: enumerate then delete by primary _id (delete_by_query is
- # silently filtered under DLS).
+ # stored under ids {file_hash}_{i}; if the new chunk count is lower
+ # than the prior one, trailing chunks would otherwise survive the
+ # writer's idempotent upsert.
+ # DLS-safe: enumerate visible chunk ids with the scoped user client,
+ # then delete concrete ids with the trusted backend client.
try:
from utils.opensearch_delete import (
collect_visible_document_ids,
delete_document_ids,
)
+ write_client = clients.opensearch
+ if write_client is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
+
stale_chunk_ids = await collect_visible_document_ids(
opensearch_client,
index=get_index_name(),
query={"term": {"document_id": file_hash}},
)
await delete_document_ids(
- opensearch_client,
+ write_client,
index=get_index_name(),
document_ids=stale_chunk_ids,
refresh=True,
@@ -348,62 +374,48 @@ class TaskProcessor:
error=str(e),
)
- # Index each chunk as a separate document
- for i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings, strict=True)):
- chunk_doc = {
- "document_id": file_hash,
- "filename": original_filename if original_filename else slim_doc["filename"],
- "mimetype": slim_doc["mimetype"],
- "page": chunk["page"],
- "text": chunk["text"],
- # Store embedding in model-specific field
- embedding_field_name: vect,
- # Track which model was used
- "embedding_model": embedding_model,
- "embedding_dimensions": len(vect),
- "file_size": file_size,
- "connector_type": connector_type,
- "indexed_time": datetime.datetime.now().isoformat(),
- }
- if connector_file_id:
- chunk_doc["connector_file_id"] = connector_file_id
-
- # Set owner and ACL fields.
- # owner is always the syncing/uploading user (matching the Langflow
- # pipeline, which indexes owner from the OWNER global var = user id).
- # acl.owner (e.g. the SharePoint file's author) is intentionally NOT
- # used here — read access comes from allowed_users/allowed_groups + DLS,
- # while owner gates deletion and "my documents" ownership.
- chunk_doc["owner"] = owner_user_id
- if acl:
- # Use ACL access lists if provided (from connector)
- chunk_doc["allowed_users"] = acl.allowed_users
- chunk_doc["allowed_groups"] = acl.allowed_groups
- else:
- # No ACL provided
- chunk_doc["allowed_users"] = []
- chunk_doc["allowed_groups"] = []
-
- # Set owner metadata fields (for display)
- if owner_name is not None:
- chunk_doc["owner_name"] = owner_name
- if owner_email is not None:
- chunk_doc["owner_email"] = owner_email
-
- # Mark as sample data if specified
- if is_sample_data:
- chunk_doc["is_sample_data"] = "true"
- chunk_id = f"{file_hash}_{i}"
- try:
- await opensearch_client.index(index=get_index_name(), id=chunk_id, body=chunk_doc)
- except Exception as e:
- logger.error(
- "OpenSearch indexing failed for chunk",
- chunk_id=chunk_id,
- error=str(e),
- )
- logger.error("Chunk document details", chunk_doc=chunk_doc)
- raise
+ # Owner is always the authenticated uploading/syncing user. Upstream ACL
+ # owners/authors only contribute read access through allowed principals.
+ owner = owner_user_id
+ if acl:
+ allowed_users = acl.allowed_users or []
+ allowed_groups = acl.allowed_groups or []
+ allowed_principals = acl.allowed_principals or []
+ allowed_principal_labels = acl.allowed_principal_labels or []
+ else:
+ allowed_users = []
+ allowed_groups = []
+ allowed_principals = []
+ allowed_principal_labels = []
+
+ filename = original_filename if original_filename else slim_doc["filename"]
+ index_context = DocumentIndexContext(
+ document_id=file_hash,
+ filename=filename,
+ mimetype=slim_doc["mimetype"],
+ embedding_model=embedding_model,
+ owner=owner,
+ owner_name=owner_name,
+ owner_email=owner_email,
+ file_size=file_size,
+ connector_type=connector_type,
+ allowed_users=allowed_users,
+ allowed_groups=allowed_groups,
+ allowed_principals=allowed_principals,
+ allowed_principal_labels=allowed_principal_labels,
+ is_sample_data=is_sample_data,
+ )
+ index_chunks = [
+ DocumentIndexChunk(
+ chunk_id=f"{file_hash}_{i}",
+ text=chunk["text"],
+ vector=vect,
+ page=chunk["page"],
+ metadata={"connector_file_id": connector_file_id} if connector_file_id else {},
+ )
+ for i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings, strict=True))
+ ]
+ await document_index_writer.index_chunks(index_context, index_chunks, final=True)
return {"status": "indexed", "id": file_hash}
async def process_item(self, upload_task: UploadTask, item: Any, file_task: FileTask) -> None:
@@ -873,7 +885,11 @@ class ConnectorFileProcessor(TaskProcessor):
# Update indexed chunks with connector-specific metadata
if result["status"] in ["indexed", "unchanged"]:
await self.connector_service._update_connector_metadata(
- document, self.user_id, connection.connector_type, self.jwt_token
+ document,
+ self.user_id,
+ connection.connector_type,
+ self.jwt_token,
+ id_field="connector_file_id",
)
# Add connector-specific metadata
@@ -928,6 +944,10 @@ class S3FileProcessor(TaskProcessor):
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Download an S3 object and process it using DocumentService"""
+ import time
+
+ from models.tasks import TaskStatus
+
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
@@ -1038,10 +1058,12 @@ class LangflowFileProcessor(TaskProcessor):
elif filename_exists and self.replace_duplicates:
# Delete existing document before uploading new one
logger.info(f"Replacing existing document: {original_filename}")
- await self.delete_document_by_filename(original_filename, opensearch_client)
- # Refresh index to make deletion visible before processing
- from config.settings import get_index_name
-
+ await self.delete_document_by_filename(
+ original_filename,
+ opensearch_client,
+ owner_user_id=self.owner_user_id,
+ )
+ # Refresh index to make deletion visible before processing.
try:
await opensearch_client.indices.refresh(index=get_index_name())
except Exception as refresh_error:
@@ -1066,16 +1088,12 @@ class LangflowFileProcessor(TaskProcessor):
)
file_tuple = (langflow_filename, content, content_type)
- # Get JWT token using same logic as DocumentFileProcessor
- # This will handle anonymous JWT creation if needed
effective_jwt = self.jwt_token
if self.session_manager and not effective_jwt:
- # Let session manager handle anonymous JWT creation if needed
- self.session_manager.get_user_opensearch_client(self.owner_user_id, self.jwt_token)
- # The session manager would have created anonymous JWT if needed
- # Get it from the session manager's internal state
- if hasattr(self.session_manager, "_anonymous_jwt"):
- effective_jwt = self.session_manager._anonymous_jwt
+ effective_jwt = self.session_manager.get_effective_jwt_token(
+ self.owner_user_id,
+ None,
+ )
# Prepare metadata tweaks similar to API endpoint
final_tweaks = self.tweaks.copy() if self.tweaks else {}
diff --git a/src/models/url.py b/src/models/url.py
index e51e5a6bd..7dbb16111 100644
--- a/src/models/url.py
+++ b/src/models/url.py
@@ -1,10 +1,10 @@
-import asyncio
import time
-from typing import Any, Dict, Optional
+from typing import Any
from utils.logging_config import get_logger
+
from .processors import TaskProcessor
-from .tasks import UploadTask, FileTask, TaskStatus
+from .tasks import FileTask, TaskStatus, UploadTask
logger = get_logger(__name__)
@@ -24,7 +24,7 @@ class LangflowUrlProcessor(TaskProcessor):
owner_email: str = None,
connector_type: str = "openrag_docs",
prevent_outside: bool = True,
- tweaks: Optional[Dict[str, Any]] = None,
+ tweaks: dict[str, Any] | None = None,
):
super().__init__()
self.langflow_file_service = langflow_file_service
@@ -86,9 +86,7 @@ class LangflowUrlProcessor(TaskProcessor):
)
return int(response.get("count", 0))
- async def process_item(
- self, upload_task: UploadTask, item: str, file_task: FileTask
- ) -> None:
+ async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
"""Process one URL ingestion item through Langflow."""
file_task.status = TaskStatus.RUNNING
file_task.updated_at = time.time()
@@ -96,11 +94,10 @@ class LangflowUrlProcessor(TaskProcessor):
try:
effective_jwt = self.jwt_token
if self.session_manager and not effective_jwt:
- self.session_manager.get_user_opensearch_client(
- self.owner_user_id, self.jwt_token
+ effective_jwt = self.session_manager.get_effective_jwt_token(
+ self.owner_user_id,
+ None,
)
- if hasattr(self.session_manager, "_anonymous_jwt"):
- effective_jwt = self.session_manager._anonymous_jwt
opensearch_client = self.session_manager.get_user_opensearch_client(
self.owner_user_id, effective_jwt
diff --git a/src/services/api_key_service.py b/src/services/api_key_service.py
index ba33eb156..92879cb58 100644
--- a/src/services/api_key_service.py
+++ b/src/services/api_key_service.py
@@ -1,15 +1,18 @@
"""
API Key Service for managing user API keys for public API authentication.
"""
+
import hashlib
+import hmac
import secrets
from datetime import datetime
-from typing import Any, Dict, List, Optional
+from typing import Any
from config.settings import API_KEYS_INDEX_NAME
from utils.logging_config import get_logger
logger = get_logger(__name__)
+API_KEY_HASH_PREFIX = "hmac-sha256:"
class APIKeyService:
@@ -25,7 +28,7 @@ class APIKeyService:
Returns:
Tuple of (full_key, key_hash, key_prefix)
- full_key: The complete API key to return to user (only shown once)
- - key_hash: SHA-256 hash of the key for storage
+ - key_hash: keyed HMAC digest of the key for storage
- key_prefix: First 12 chars for display (e.g., "orag_abc12345")
"""
# Generate 32 bytes of random data, encode as base64url (no padding)
@@ -34,8 +37,7 @@ class APIKeyService:
# Create the full key with prefix
full_key = f"orag_{random_bytes}"
- # Hash the full key for storage
- key_hash = hashlib.sha256(full_key.encode()).hexdigest()
+ key_hash = self._hash_key(full_key)
# Create prefix for display (orag_ + first 8 chars of random part)
key_prefix = f"orag_{random_bytes[:8]}"
@@ -45,17 +47,45 @@ class APIKeyService:
def _get_opensearch_client(self, jwt_token: str = None):
"""Get the appropriate OpenSearch client.
- In IBM auth mode, returns a user-authenticated client when *jwt_token*
- is available; otherwise falls back to the default admin client.
+ Upstream-authenticated requests can use the user credential for scoped
+ reads. Writes go through the backend client so the OpenSearch user role
+ can stay read-only.
"""
from config.settings import IBM_AUTH_ENABLED, clients
+
if IBM_AUTH_ENABLED and jwt_token and self.session_manager:
return clients.create_user_opensearch_client(jwt_token)
return clients.opensearch
+ def _get_write_opensearch_client(self):
+ """Return the trusted backend OpenSearch client for API-key writes."""
+ from config.settings import clients
+
+ if clients.opensearch is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
+ return clients.opensearch
+
def _hash_key(self, api_key: str) -> str:
- """Hash an API key for lookup."""
- return hashlib.sha256(api_key.encode()).hexdigest()
+ """Create the keyed lookup digest stored in OpenSearch."""
+ from config.settings import SESSION_SECRET
+
+ digest = hmac.digest(
+ SESSION_SECRET.encode("utf-8"),
+ api_key.encode("utf-8"),
+ "sha256",
+ ).hex()
+ return f"{API_KEY_HASH_PREFIX}{digest}"
+
+ def _legacy_hash_key(self, api_key: str) -> str:
+ """Return the pre-HMAC lookup digest for backwards compatibility."""
+ digest = hashlib.new("sha256", usedforsecurity=False) # nosec B324
+ digest.update(api_key.encode("utf-8"))
+ return digest.hexdigest()
+
+ def _candidate_hashes(self, api_key: str) -> list[str]:
+ keyed_hash = self._hash_key(api_key)
+ legacy_hash = self._legacy_hash_key(api_key)
+ return [keyed_hash, legacy_hash]
async def create_key(
self,
@@ -63,7 +93,7 @@ class APIKeyService:
user_email: str,
name: str,
jwt_token: str = None,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""
Create a new API key for a user.
@@ -98,7 +128,7 @@ class APIKeyService:
"revoked": False,
}
- opensearch_client = self._get_opensearch_client(jwt_token)
+ opensearch_client = self._get_write_opensearch_client()
# Index the key document
result = await opensearch_client.index(
@@ -130,7 +160,7 @@ class APIKeyService:
logger.error("Failed to create API key", error=str(e), user_id=user_id)
return {"success": False, "error": str(e)}
- async def validate_key(self, api_key: str) -> Optional[Dict[str, Any]]:
+ async def validate_key(self, api_key: str) -> dict[str, Any] | None:
"""
Validate an API key and return user info if valid.
@@ -145,8 +175,8 @@ class APIKeyService:
if not api_key or not api_key.startswith("orag_"):
return None
- # Hash the incoming key
key_hash = self._hash_key(api_key)
+ candidate_hashes = self._candidate_hashes(api_key)
opensearch_client = self._get_opensearch_client()
@@ -155,7 +185,7 @@ class APIKeyService:
"query": {
"bool": {
"must": [
- {"term": {"key_hash": key_hash}},
+ {"terms": {"key_hash": candidate_hashes}},
{"term": {"revoked": False}},
]
}
@@ -174,16 +204,18 @@ class APIKeyService:
key_doc = hits[0]["_source"]
- # Update last_used_at (fire and forget)
+ matched_hash = key_doc.get("key_hash")
+
+ # Update last_used_at and opportunistically migrate legacy hashes.
try:
- await opensearch_client.update(
+ write_client = self._get_write_opensearch_client()
+ update_doc = {"last_used_at": datetime.utcnow().isoformat()}
+ if matched_hash != key_hash:
+ update_doc["key_hash"] = key_hash
+ await write_client.update(
index=API_KEYS_INDEX_NAME,
id=key_doc["key_id"],
- body={
- "doc": {
- "last_used_at": datetime.utcnow().isoformat()
- }
- },
+ body={"doc": update_doc},
)
except Exception:
pass # Don't fail validation if update fails
@@ -203,7 +235,7 @@ class APIKeyService:
self,
user_id: str,
jwt_token: str = None,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""
List all API keys for a user (without the actual keys).
@@ -219,9 +251,7 @@ class APIKeyService:
# Search for user's keys
search_body = {
- "query": {
- "term": {"user_id": user_id}
- },
+ "query": {"term": {"user_id": user_id}},
"sort": [{"created_at": {"order": "desc"}}],
"_source": [
"key_id",
@@ -254,7 +284,7 @@ class APIKeyService:
user_id: str,
key_id: str,
jwt_token: str = None,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""
Revoke an API key.
@@ -282,15 +312,12 @@ class APIKeyService:
except Exception:
return {"success": False, "error": "Key not found"}
- # Update the key to mark as revoked
- result = await opensearch_client.update(
+ # Update the key to mark as revoked with the trusted backend client.
+ write_client = self._get_write_opensearch_client()
+ result = await write_client.update(
index=API_KEYS_INDEX_NAME,
id=key_id,
- body={
- "doc": {
- "revoked": True
- }
- },
+ body={"doc": {"revoked": True}},
refresh="wait_for",
)
@@ -318,7 +345,7 @@ class APIKeyService:
user_id: str,
key_id: str,
jwt_token: str = None,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""
Permanently delete an API key.
@@ -346,8 +373,9 @@ class APIKeyService:
except Exception:
return {"success": False, "error": "Key not found"}
- # Delete the key
- result = await opensearch_client.delete(
+ # Delete the key with the trusted backend client.
+ write_client = self._get_write_opensearch_client()
+ result = await write_client.delete(
index=API_KEYS_INDEX_NAME,
id=key_id,
refresh="wait_for",
diff --git a/src/services/default_docs_service.py b/src/services/default_docs_service.py
index 2b5774e7e..a28f6271e 100644
--- a/src/services/default_docs_service.py
+++ b/src/services/default_docs_service.py
@@ -150,7 +150,7 @@ async def ingest_default_documents_when_ready(
task_service,
file_paths,
existing_task_id=task_id,
- connector_type="local",
+ connector_type="openrag_docs",
jwt_token=jwt_token,
)
task_id = new_task_id or task_id
@@ -161,7 +161,7 @@ async def ingest_default_documents_when_ready(
task_service,
file_paths,
existing_task_id=task_id,
- connector_type="local",
+ connector_type="openrag_docs",
jwt_token=jwt_token,
)
task_id = new_task_id or task_id
@@ -202,9 +202,7 @@ async def _ingest_default_documents_langflow(
effective_jwt = jwt_token
if not effective_jwt and session_manager:
- session_manager.get_user_opensearch_client(anonymous_user.user_id, effective_jwt)
- if hasattr(session_manager, "_anonymous_jwt"):
- effective_jwt = session_manager._anonymous_jwt
+ effective_jwt = session_manager.get_effective_jwt_token(anonymous_user.user_id, None)
default_tweaks = {
"OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4": {
@@ -267,9 +265,7 @@ async def _ingest_default_documents_url_langflow(
effective_jwt = jwt_token
if not effective_jwt and session_manager:
- session_manager.get_user_opensearch_client(anonymous_user.user_id, effective_jwt)
- if hasattr(session_manager, "_anonymous_jwt"):
- effective_jwt = session_manager._anonymous_jwt
+ effective_jwt = session_manager.get_effective_jwt_token(anonymous_user.user_id, None)
default_tweaks = {
"OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4": {
@@ -368,7 +364,13 @@ async def _ingest_default_documents_url(
async def _delete_existing_default_docs(session_manager, connector_type: str, jwt_token=None):
"""Delete previously ingested default OpenRAG docs before reingestion."""
+ from config.settings import clients
from session_manager import AnonymousUser
+ from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
+
+ write_client = clients.opensearch
+ if write_client is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
if session_manager is None:
logger.warning(
@@ -379,9 +381,7 @@ async def _delete_existing_default_docs(session_manager, connector_type: str, jw
anonymous_user = AnonymousUser()
effective_jwt = jwt_token
if not effective_jwt and session_manager:
- session_manager.get_user_opensearch_client(anonymous_user.user_id, effective_jwt)
- if hasattr(session_manager, "_anonymous_jwt"):
- effective_jwt = session_manager._anonymous_jwt
+ effective_jwt = session_manager.get_effective_jwt_token(anonymous_user.user_id, None)
opensearch_client = session_manager.get_user_opensearch_client(
anonymous_user.user_id, effective_jwt
@@ -405,14 +405,20 @@ async def _delete_existing_default_docs(session_manager, connector_type: str, jw
}
}
}
- result = await opensearch_client.delete_by_query(
- index=get_index_name(),
- body=delete_query,
- conflicts="proceed",
+ index_name = get_index_name()
+ document_ids = await collect_visible_document_ids(
+ opensearch_client,
+ index=index_name,
+ query=delete_query["query"],
+ )
+ deleted_chunks = await delete_document_ids(
+ write_client,
+ index=index_name,
+ document_ids=document_ids,
)
logger.info(
"Deleted existing default OpenRAG docs before reingestion",
- deleted_chunks=result.get("deleted", 0),
+ deleted_chunks=deleted_chunks,
)
diff --git a/src/services/dls_principal_service.py b/src/services/dls_principal_service.py
new file mode 100644
index 000000000..07cfa73ec
--- /dev/null
+++ b/src/services/dls_principal_service.py
@@ -0,0 +1,413 @@
+"""Maintain OpenSearch DLS lookup rows for connector ACL principals."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from datetime import UTC, datetime
+from typing import Any
+
+from session_manager import User
+from utils.group_acl import (
+ acl_principal_label,
+ unique_acl_principal_labels,
+ unique_acl_principals,
+)
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class DLSPrincipalService:
+ """Refresh the per-OpenSearch-user principal lookup index.
+
+ Document DLS can only see the OpenSearch authenticated principal and roles.
+ This service bridges connector-specific user aliases by writing a lookup row
+ keyed by the actual OpenSearch user name. DLS then uses a terms lookup on the
+ row's ``principals`` array.
+ """
+
+ def __init__(
+ self,
+ connector_service: Any,
+ opensearch_client: Any | None = None,
+ refresh_ttl_seconds: int | None = None,
+ ):
+ self.connector_service = connector_service
+ self.opensearch_client = opensearch_client
+ if refresh_ttl_seconds is None:
+ from config.settings import DLS_PRINCIPAL_REFRESH_TTL_SECONDS
+
+ refresh_ttl_seconds = DLS_PRINCIPAL_REFRESH_TTL_SECONDS
+ self.refresh_ttl_seconds = max(refresh_ttl_seconds, 0)
+ self._admin_opensearch_client: Any | None = None
+ self._ensure_lock = asyncio.Lock()
+ self._index_checked = False
+ self._cache: dict[str, tuple[float, list[str]]] = {}
+ self._locks: dict[str, asyncio.Lock] = {}
+
+ async def refresh_user_principals(
+ self,
+ user: User | None,
+ *,
+ group_roles: list[str] | None = None,
+ ) -> list[str]:
+ """Resolve and persist current DLS principals for this request user."""
+ if user is None or not user.user_id:
+ return []
+
+ if self.refresh_ttl_seconds <= 0:
+ return await self._refresh_user_principals_uncached(user, group_roles=group_roles)
+
+ cache_key = self._refresh_cache_key(user, group_roles)
+ if self.refresh_ttl_seconds > 0:
+ cached = self._cache.get(cache_key)
+ now = time.monotonic()
+ if cached and cached[0] > now:
+ return list(cached[1])
+
+ lock = self._locks.setdefault(cache_key, asyncio.Lock())
+ async with lock:
+ if self.refresh_ttl_seconds > 0:
+ cached = self._cache.get(cache_key)
+ now = time.monotonic()
+ if cached and cached[0] > now:
+ return list(cached[1])
+
+ return await self._refresh_user_principals_uncached(user, group_roles=group_roles)
+
+ async def _refresh_user_principals_uncached(
+ self,
+ user: User,
+ *,
+ group_roles: list[str] | None,
+ ) -> list[str]:
+ user_names = self._opensearch_user_names(user)
+ if not user_names:
+ return []
+
+ client = self._get_opensearch_client()
+ if client is None:
+ logger.warning(
+ "Unable to refresh DLS principals: OpenSearch client is unavailable",
+ user_id=user.user_id,
+ )
+ return []
+
+ try:
+ await self.ensure_index(client)
+ except Exception as e:
+ logger.warning(
+ "Failed to prepare DLS principal lookup index",
+ user_id=user.user_id,
+ user_names=user_names,
+ error=str(e),
+ )
+ return []
+
+ (
+ connector_principals,
+ connector_labels,
+ ) = await self._resolve_connector_principals_and_labels(
+ user,
+ include_group_roles=group_roles is None,
+ )
+ auth_principals = self._resolve_auth_user_principals(user)
+ principals = unique_acl_principals(
+ [
+ *(group_roles or []),
+ *connector_principals,
+ *auth_principals,
+ ]
+ )
+ principal_labels = unique_acl_principal_labels(
+ [
+ *connector_labels,
+ *self._labels_for_group_roles(group_roles or []),
+ *self._labels_for_auth_user_principals(user, auth_principals),
+ ]
+ )
+
+ try:
+ updated_at = datetime.now(UTC).isoformat()
+ for user_name in user_names:
+ await client.index(
+ index=self.index_name,
+ id=user_name,
+ body={
+ "user_name": user_name,
+ "auth_user_id": user.user_id,
+ "auth_email": user.email,
+ "provider": user.provider,
+ "principals": principals,
+ "principal_labels": principal_labels,
+ "updated_at": updated_at,
+ },
+ refresh="wait_for",
+ )
+ self._cache_principals(user, group_roles, principals)
+ except Exception as e:
+ logger.warning(
+ "Failed to refresh DLS principal lookup row",
+ user_id=user.user_id,
+ user_names=user_names,
+ principal_count=len(principals),
+ error=str(e),
+ )
+
+ return principals
+
+ def invalidate_user(self, user: User | str) -> None:
+ """Drop cached DLS principals for one auth user or OpenSearch user name."""
+ user_id = user.user_id if isinstance(user, User) else user
+ keys = [key for key in self._cache if user_id in key.split("\x1f")]
+ for key in keys:
+ self._cache.pop(key, None)
+ self._locks.pop(key, None)
+
+ def clear(self) -> None:
+ self._cache.clear()
+ self._locks.clear()
+
+ def _cache_principals(
+ self,
+ user: User,
+ group_roles: list[str] | None,
+ principals: list[str],
+ ) -> None:
+ if self.refresh_ttl_seconds <= 0:
+ return
+ self._cache[self._refresh_cache_key(user, group_roles)] = (
+ time.monotonic() + self.refresh_ttl_seconds,
+ list(principals),
+ )
+
+ def _refresh_cache_key(self, user: User, group_roles: list[str] | None) -> str:
+ key_parts = [
+ user.user_id,
+ user.opensearch_username,
+ user.email,
+ user.provider,
+ ]
+ if group_roles is not None:
+ key_parts.extend(sorted(group_roles))
+ return "\x1f".join(str(part) for part in key_parts if part)
+
+ @property
+ def index_name(self) -> str:
+ from config.settings import DLS_PRINCIPAL_INDEX_NAME
+
+ return DLS_PRINCIPAL_INDEX_NAME
+
+ async def ensure_index(self, client: Any) -> None:
+ """Create the lookup index if it does not exist."""
+ if self._index_checked:
+ return
+
+ async with self._ensure_lock:
+ if self._index_checked:
+ return
+
+ from config.settings import DLS_PRINCIPAL_INDEX_BODY
+
+ if not await client.indices.exists(index=self.index_name):
+ await client.indices.create(index=self.index_name, body=DLS_PRINCIPAL_INDEX_BODY)
+ logger.info("Created DLS principal lookup index", index_name=self.index_name)
+ else:
+ try:
+ mapping = await client.indices.get_mapping(index=self.index_name)
+ properties = (
+ mapping.get(self.index_name, {}).get("mappings", {}).get("properties", {})
+ )
+ if properties.get("principal_labels") is None:
+ await client.indices.put_mapping(
+ index=self.index_name,
+ body={
+ "properties": {
+ "principal_labels": DLS_PRINCIPAL_INDEX_BODY["mappings"][
+ "properties"
+ ]["principal_labels"]
+ }
+ },
+ )
+ except AttributeError:
+ logger.debug(
+ "Skipping DLS principal label mapping backfill; client does not expose mapping APIs",
+ index_name=self.index_name,
+ )
+ self._index_checked = True
+
+ def _get_opensearch_client(self) -> Any | None:
+ if self.opensearch_client is not None:
+ return self.opensearch_client
+
+ try:
+ from config.settings import (
+ IBM_AUTH_ENABLED,
+ OPENSEARCH_PASSWORD,
+ OPENSEARCH_USERNAME,
+ clients,
+ )
+
+ if IBM_AUTH_ENABLED:
+ if not OPENSEARCH_PASSWORD:
+ return None
+ if self._admin_opensearch_client is None:
+ self._admin_opensearch_client = clients.create_basic_opensearch_client(
+ OPENSEARCH_USERNAME,
+ OPENSEARCH_PASSWORD,
+ )
+ return self._admin_opensearch_client
+
+ return clients.opensearch
+ except Exception:
+ return None
+
+ @staticmethod
+ def _opensearch_user_names(user: User) -> list[str]:
+ return unique_acl_principals(
+ [
+ user.opensearch_username,
+ user.user_id,
+ ]
+ )
+
+ def _resolve_auth_user_principals(self, user: User) -> list[str]:
+ connection_manager = getattr(self.connector_service, "connection_manager", None)
+ resolver = getattr(connection_manager, "get_auth_user_principals", None)
+ if resolver is None:
+ return []
+ try:
+ return resolver(user) or []
+ except Exception as e:
+ logger.warning(
+ "Failed to resolve auth-user DLS principals",
+ user_id=user.user_id,
+ error=str(e),
+ )
+ return []
+
+ async def _resolve_connector_principals_and_labels(
+ self,
+ user: User,
+ *,
+ include_group_roles: bool,
+ ) -> tuple[list[str], list[dict[str, Any]]]:
+ connection_manager = getattr(self.connector_service, "connection_manager", None)
+ if connection_manager is None:
+ return [], []
+
+ try:
+ connections = await connection_manager.list_connections(user_id=user.user_id)
+ except Exception as e:
+ logger.warning("Failed to list connector connections for DLS principals", error=str(e))
+ return [], []
+
+ principals: list[str] = []
+ labels: list[dict[str, Any]] = []
+ for connection in connections:
+ if not getattr(connection, "is_active", False):
+ continue
+
+ try:
+ connector = await self.connector_service.get_connector(connection.connection_id)
+ except Exception as e:
+ logger.debug(
+ "Skipping connector DLS principal lookup",
+ connection_id=getattr(connection, "connection_id", None),
+ connector_type=getattr(connection, "connector_type", None),
+ error=str(e),
+ )
+ continue
+
+ if connector is None:
+ continue
+
+ label_resolver = getattr(connector, "get_current_user_principal_labels", None)
+ if label_resolver is not None:
+ try:
+ labels.extend(await label_resolver() or [])
+ except NotImplementedError:
+ pass
+ except Exception as e:
+ logger.warning(
+ "Connector DLS principal label lookup failed",
+ connection_id=getattr(connection, "connection_id", None),
+ connector_type=getattr(connection, "connector_type", None),
+ error=str(e),
+ )
+
+ try:
+ principals.extend(await connector.get_current_user_principals() or [])
+ except NotImplementedError:
+ pass
+ except Exception as e:
+ logger.warning(
+ "Connector DLS principal lookup failed",
+ connection_id=getattr(connection, "connection_id", None),
+ connector_type=getattr(connection, "connector_type", None),
+ error=str(e),
+ )
+
+ if not include_group_roles:
+ continue
+
+ try:
+ principals.extend(await connector.get_current_user_group_roles() or [])
+ except NotImplementedError:
+ pass
+ except Exception as e:
+ logger.warning(
+ "Connector group principal lookup failed",
+ connection_id=getattr(connection, "connection_id", None),
+ connector_type=getattr(connection, "connector_type", None),
+ error=str(e),
+ )
+
+ return principals, labels
+
+ @staticmethod
+ def _principal_label_kind(principal: str) -> str:
+ if principal.startswith("g:"):
+ return "group"
+ if principal.startswith("u:"):
+ return "user"
+ return "unknown"
+
+ @staticmethod
+ def _principal_label_provider(principal: str) -> str:
+ parts = principal.split(":", 2)
+ return parts[1] if len(parts) > 1 and parts[1] else "unknown"
+
+ def _labels_for_group_roles(self, group_roles: list[str]) -> list[dict[str, Any]]:
+ labels: list[dict[str, Any]] = []
+ for role in group_roles:
+ label = acl_principal_label(
+ role,
+ kind="group",
+ provider=self._principal_label_provider(role),
+ display_name=role,
+ external_id=role,
+ )
+ if label:
+ labels.append(label)
+ return labels
+
+ def _labels_for_auth_user_principals(
+ self,
+ user: User,
+ auth_principals: list[str],
+ ) -> list[dict[str, Any]]:
+ labels: list[dict[str, Any]] = []
+ for principal in auth_principals:
+ label = acl_principal_label(
+ principal,
+ kind=self._principal_label_kind(principal),
+ provider=self._principal_label_provider(principal),
+ display_name=user.name or user.email or principal,
+ email=user.email,
+ external_id=user.user_id,
+ )
+ if label:
+ labels.append(label)
+ return labels
diff --git a/src/services/document_index_writer.py b/src/services/document_index_writer.py
new file mode 100644
index 000000000..95387c523
--- /dev/null
+++ b/src/services/document_index_writer.py
@@ -0,0 +1,277 @@
+"""Shared backend-owned OpenSearch document indexing.
+
+Langflow can generate chunks and embeddings, but it must not hold credentials
+that can write arbitrary documents. This writer is the single backend path for
+indexing chunks into the documents index.
+"""
+
+from __future__ import annotations
+
+import datetime
+import json
+from dataclasses import dataclass, field
+from typing import Any
+
+from utils.embedding_fields import ensure_embedding_field_exists
+from utils.embeddings import create_index_body
+from utils.group_acl import unique_acl_principal_labels, unique_acl_principals
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+@dataclass
+class DocumentIndexContext:
+ document_id: str
+ filename: str
+ mimetype: str
+ embedding_model: str
+ owner: str | None = None
+ owner_name: str | None = None
+ owner_email: str | None = None
+ file_size: int | None = None
+ connector_type: str | None = None
+ source_url: str | None = None
+ allowed_users: list[str] = field(default_factory=list)
+ allowed_groups: list[str] = field(default_factory=list)
+ allowed_principals: list[str] = field(default_factory=list)
+ allowed_principal_labels: list[dict[str, Any]] = field(default_factory=list)
+ ingest_run_id: str | None = None
+ is_sample_data: bool = False
+ index_name: str | None = None
+
+
+@dataclass
+class DocumentIndexChunk:
+ chunk_id: str
+ text: str
+ vector: list[float]
+ page: int | None = None
+ metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class DocumentIndexWriter:
+ """Write document chunks with a trusted backend OpenSearch client."""
+
+ def __init__(self, opensearch_client: Any | None = None):
+ self.opensearch_client = opensearch_client
+
+ def _get_write_client(self) -> Any:
+ from config.settings import clients
+
+ client = self.opensearch_client or clients.opensearch
+ if client is None:
+ raise RuntimeError(
+ "Backend OpenSearch write client is unavailable; cannot index document chunks"
+ )
+ return client
+
+ async def index_chunks(
+ self,
+ context: DocumentIndexContext,
+ chunks: list[DocumentIndexChunk],
+ *,
+ final: bool = False,
+ refresh: bool | str = False,
+ ) -> dict[str, Any]:
+ """Index one batch of chunks.
+
+ Repeated calls with the same chunk ids are idempotent because the write
+ operation is an index/upsert.
+ """
+ from config.settings import get_index_name
+
+ if not chunks:
+ if final:
+ await self._refresh(context.index_name or get_index_name())
+ return {"indexed_chunks": 0, "ingest_run_id": context.ingest_run_id}
+
+ first_vector = chunks[0].vector
+ if not first_vector:
+ raise ValueError("Cannot index chunks with empty embeddings")
+
+ dimensions = len(first_vector)
+ client = self._get_write_client()
+ index_name = context.index_name or get_index_name()
+ embedding_field = await self._ensure_index_and_embedding_field(
+ client,
+ index_name=index_name,
+ embedding_model=context.embedding_model,
+ dimensions=dimensions,
+ )
+
+ now = datetime.datetime.now(datetime.UTC).isoformat()
+ bulk_body: list[dict[str, Any]] = []
+ for chunk in chunks:
+ if len(chunk.vector) != dimensions:
+ raise ValueError(
+ "Embedding dimension mismatch in batch: "
+ f"expected {dimensions}, got {len(chunk.vector)} for {chunk.chunk_id}"
+ )
+ bulk_body.append({"index": {"_index": index_name, "_id": chunk.chunk_id}})
+ bulk_body.append(
+ self._build_chunk_document(
+ context=context,
+ chunk=chunk,
+ embedding_field=embedding_field,
+ indexed_time=now,
+ )
+ )
+
+ result = await client.bulk(body=bulk_body, refresh=refresh)
+ self._raise_for_bulk_errors(result)
+ if final:
+ await self._refresh(index_name)
+
+ logger.info(
+ "Indexed document chunks",
+ index_name=index_name,
+ document_id=context.document_id,
+ ingest_run_id=context.ingest_run_id,
+ chunk_count=len(chunks),
+ final=final,
+ )
+ return {
+ "indexed_chunks": len(chunks),
+ "ingest_run_id": context.ingest_run_id,
+ "document_id": context.document_id,
+ }
+
+ async def delete_ingest_run(self, ingest_run_id: str, *, index_name: str | None = None) -> int:
+ """Delete partially indexed chunks for a failed callback run."""
+ if not ingest_run_id:
+ return 0
+ from config.settings import get_index_name
+
+ client = self._get_write_client()
+ resolved_index = index_name or get_index_name()
+ body = {"query": {"term": {"ingest_run_id": ingest_run_id}}}
+ response = await client.delete_by_query(
+ index=resolved_index,
+ body=body,
+ refresh=True,
+ conflicts="proceed",
+ )
+ deleted = int(response.get("deleted", 0)) if isinstance(response, dict) else 0
+ logger.info(
+ "Deleted failed ingest run chunks",
+ index_name=resolved_index,
+ ingest_run_id=ingest_run_id,
+ deleted=deleted,
+ )
+ return deleted
+
+ async def _ensure_index_and_embedding_field(
+ self,
+ client: Any,
+ *,
+ index_name: str,
+ embedding_model: str,
+ dimensions: int,
+ ) -> str:
+ if not await client.indices.exists(index=index_name):
+ await client.indices.create(
+ index=index_name,
+ body=await create_index_body(embedding_model, dimensions),
+ )
+ return await ensure_embedding_field_exists(
+ client,
+ embedding_model,
+ index_name,
+ dimensions,
+ )
+
+ def _build_chunk_document(
+ self,
+ *,
+ context: DocumentIndexContext,
+ chunk: DocumentIndexChunk,
+ embedding_field: str,
+ indexed_time: str,
+ ) -> dict[str, Any]:
+ metadata = self._normalized_metadata(chunk.metadata)
+ document_id = context.document_id or str(metadata.get("document_id") or chunk.chunk_id)
+ filename = context.filename or str(metadata.get("filename") or "")
+ mimetype = context.mimetype or str(metadata.get("mimetype") or "")
+
+ doc: dict[str, Any] = {
+ "document_id": document_id,
+ "filename": filename,
+ "mimetype": mimetype,
+ "page": chunk.page if chunk.page is not None else metadata.get("page", 0),
+ "text": chunk.text,
+ embedding_field: chunk.vector,
+ "embedding_model": context.embedding_model,
+ "embedding_dimensions": len(chunk.vector),
+ "file_size": context.file_size
+ if context.file_size is not None
+ else metadata.get("file_size"),
+ "connector_type": context.connector_type or metadata.get("connector_type") or "local",
+ "source_url": context.source_url or metadata.get("source_url") or "",
+ "owner": context.owner,
+ "allowed_users": list(context.allowed_users),
+ "allowed_groups": list(context.allowed_groups),
+ "allowed_principals": unique_acl_principals(context.allowed_principals),
+ "allowed_principal_labels": unique_acl_principal_labels(
+ context.allowed_principal_labels
+ ),
+ "indexed_time": indexed_time,
+ "metadata": metadata.get("metadata", {}),
+ }
+
+ if context.owner_name is not None:
+ doc["owner_name"] = context.owner_name
+ if context.owner_email is not None:
+ doc["owner_email"] = context.owner_email
+ if context.ingest_run_id:
+ doc["ingest_run_id"] = context.ingest_run_id
+ if metadata.get("connector_file_id"):
+ doc["connector_file_id"] = metadata["connector_file_id"]
+ if context.is_sample_data:
+ doc["is_sample_data"] = "true"
+ for time_field in ("created_time", "modified_time"):
+ if metadata.get(time_field):
+ doc[time_field] = metadata[time_field]
+
+ return doc
+
+ @staticmethod
+ def _normalized_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
+ normalized = dict(metadata or {})
+ for key in (
+ "allowed_users",
+ "allowed_groups",
+ "allowed_principals",
+ "allowed_principal_labels",
+ ):
+ value = normalized.get(key)
+ if isinstance(value, str):
+ try:
+ parsed = json.loads(value)
+ except (TypeError, json.JSONDecodeError):
+ continue
+ if isinstance(parsed, list):
+ normalized[key] = parsed
+ if "filesize" in normalized and "file_size" not in normalized:
+ normalized["file_size"] = normalized["filesize"]
+ return normalized
+
+ @staticmethod
+ def _raise_for_bulk_errors(result: Any) -> None:
+ if not isinstance(result, dict) or not result.get("errors"):
+ return
+ failures = []
+ for item in result.get("items", [])[:5]:
+ action = item.get("index") or item.get("create") or item.get("update") or item
+ failures.append(
+ {
+ "id": action.get("_id"),
+ "status": action.get("status"),
+ "error": action.get("error"),
+ }
+ )
+ raise RuntimeError(f"OpenSearch bulk indexing failed: {failures}")
+
+ async def _refresh(self, index_name: str) -> None:
+ client = self._get_write_client()
+ await client.indices.refresh(index=index_name)
diff --git a/src/services/document_service.py b/src/services/document_service.py
index 2b920dcbf..378b5c6c1 100644
--- a/src/services/document_service.py
+++ b/src/services/document_service.py
@@ -85,10 +85,17 @@ def chunk_texts_for_embeddings(
class DocumentService:
- def __init__(self, session_manager=None, models_service=None, docling_service=None):
+ def __init__(
+ self,
+ session_manager=None,
+ models_service=None,
+ docling_service=None,
+ document_index_writer=None,
+ ):
self.session_manager = session_manager
self.models_service = models_service
self.docling_service = docling_service
+ self.document_index_writer = document_index_writer
self._mapping_ensured = False
async def process_upload_file(
diff --git a/src/services/file_service.py b/src/services/file_service.py
index d94b25160..fee5d96cd 100644
--- a/src/services/file_service.py
+++ b/src/services/file_service.py
@@ -5,7 +5,7 @@ Aggregates document chunks by filename to produce file-level views,
with support for pagination, filtering, sorting, and fuzzy search.
"""
-from typing import Any, Dict, List, Optional
+from typing import Any
from config.settings import get_index_name
from utils.logging_config import get_logger
@@ -27,11 +27,11 @@ class FileService:
page_size: int = 25,
sort_by: str = "filename",
sort_order: str = "asc",
- connector_type: Optional[str] = None,
- mimetype: Optional[str] = None,
- owner: Optional[str] = None,
- search: Optional[str] = None,
- ) -> Dict[str, Any]:
+ connector_type: str | None = None,
+ mimetype: str | None = None,
+ owner: str | None = None,
+ search: str | None = None,
+ ) -> dict[str, Any]:
"""
List ingested files with server-side pagination, filtering, and sorting.
@@ -74,10 +74,10 @@ class FileService:
query: str = "",
page: int = 1,
page_size: int = 25,
- connector_type: Optional[str] = None,
- mimetype: Optional[str] = None,
- owner: Optional[str] = None,
- ) -> Dict[str, Any]:
+ connector_type: str | None = None,
+ mimetype: str | None = None,
+ owner: str | None = None,
+ ) -> dict[str, Any]:
"""
Search files by name with fuzzy/prefix matching.
@@ -100,11 +100,11 @@ class FileService:
def _build_filter_query(
self,
user_id: str,
- connector_type: Optional[str] = None,
- mimetype: Optional[str] = None,
- owner: Optional[str] = None,
- search: Optional[str] = None,
- ) -> Dict[str, Any]:
+ connector_type: str | None = None,
+ mimetype: str | None = None,
+ owner: str | None = None,
+ search: str | None = None,
+ ) -> dict[str, Any]:
"""Build the bool query with optional filters + filename search."""
must = []
filter_clauses = []
@@ -130,13 +130,13 @@ class FileService:
}
)
- query = {"bool": {"filter": filter_clauses}}
+ query: dict[str, Any] = {"bool": {"filter": filter_clauses}}
if must:
query["bool"]["must"] = must
return query
- def _build_file_aggregation(self, query: Dict[str, Any]) -> Dict[str, Any]:
+ def _build_file_aggregation(self, query: dict[str, Any]) -> dict[str, Any]:
"""Build the OpenSearch aggregation body for file-level grouping."""
return {
"size": 0,
@@ -166,6 +166,7 @@ class FileService:
"indexed_time",
"allowed_users",
"allowed_groups",
+ "allowed_principal_labels",
],
"sort": [{"indexed_time": {"order": "desc"}}],
}
@@ -176,7 +177,7 @@ class FileService:
},
}
- def _parse_aggregation_buckets(self, result: Dict[str, Any]) -> List[Dict[str, Any]]:
+ def _parse_aggregation_buckets(self, result: dict[str, Any]) -> list[dict[str, Any]]:
"""Parse OpenSearch aggregation buckets into file dicts."""
buckets = result.get("aggregations", {}).get("files", {}).get("buckets", [])
@@ -204,6 +205,7 @@ class FileService:
"chunk_count": bucket.get("chunk_count", {}).get("value", 0),
"allowed_users": source.get("allowed_users", []),
"allowed_groups": source.get("allowed_groups", []),
+ "allowed_principal_labels": source.get("allowed_principal_labels", []),
}
)
@@ -211,10 +213,10 @@ class FileService:
def _sort_files(
self,
- files: List[Dict[str, Any]],
+ files: list[dict[str, Any]],
sort_by: str,
sort_order: str,
- ) -> List[Dict[str, Any]]:
+ ) -> list[dict[str, Any]]:
"""Sort file list by the given field."""
valid_sort_fields = {
"filename",
diff --git a/src/services/group_acl_service.py b/src/services/group_acl_service.py
new file mode 100644
index 000000000..d2a0e70ae
--- /dev/null
+++ b/src/services/group_acl_service.py
@@ -0,0 +1,117 @@
+"""Resolve connector-backed group ACL roles and principals."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import Any
+
+from session_manager import User
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class GroupACLService:
+ """Collect current-user group roles from pluggable connectors."""
+
+ def __init__(self, connector_service: Any, cache_ttl_seconds: int | None = None):
+ self.connector_service = connector_service
+ if cache_ttl_seconds is None:
+ from config.settings import GROUP_ACL_CACHE_TTL_SECONDS
+
+ cache_ttl_seconds = GROUP_ACL_CACHE_TTL_SECONDS
+ self.cache_ttl_seconds = max(cache_ttl_seconds, 0)
+ self._cache: dict[str, tuple[float, list[str]]] = {}
+ self._locks: dict[str, asyncio.Lock] = {}
+
+ async def get_user_group_roles(self, user: User | None) -> list[str]:
+ """Return OpenSearch backend roles for the user's upstream groups."""
+ if user is None or not user.user_id:
+ return []
+
+ if self.cache_ttl_seconds <= 0:
+ return await self._resolve_user_group_roles(user)
+
+ if self.cache_ttl_seconds > 0:
+ cached = self._cache.get(user.user_id)
+ now = time.monotonic()
+ if cached and cached[0] > now:
+ return list(cached[1])
+
+ lock = self._locks.setdefault(user.user_id, asyncio.Lock())
+ async with lock:
+ if self.cache_ttl_seconds > 0:
+ cached = self._cache.get(user.user_id)
+ now = time.monotonic()
+ if cached and cached[0] > now:
+ return list(cached[1])
+
+ roles = await self._resolve_user_group_roles(user)
+ if self.cache_ttl_seconds > 0:
+ self._cache[user.user_id] = (
+ time.monotonic() + self.cache_ttl_seconds,
+ list(roles),
+ )
+ return roles
+
+ def invalidate_user(self, user_id: str) -> None:
+ self._cache.pop(user_id, None)
+ self._locks.pop(user_id, None)
+
+ def clear(self) -> None:
+ self._cache.clear()
+ self._locks.clear()
+
+ async def _resolve_user_group_roles(self, user: User) -> list[str]:
+ connection_manager = getattr(self.connector_service, "connection_manager", None)
+ if connection_manager is None:
+ return []
+
+ try:
+ connections = await connection_manager.list_connections(user_id=user.user_id)
+ except Exception as e:
+ logger.warning("Failed to list connector connections for group ACLs", error=str(e))
+ return []
+
+ roles: list[str] = []
+ seen: set[str] = set()
+ for connection in connections:
+ if not getattr(connection, "is_active", False):
+ continue
+
+ connector = None
+ try:
+ connector = await self.connector_service.get_connector(connection.connection_id)
+ except Exception as e:
+ logger.debug(
+ "Skipping connector group ACL lookup",
+ connection_id=getattr(connection, "connection_id", None),
+ connector_type=getattr(connection, "connector_type", None),
+ error=str(e),
+ )
+ continue
+
+ if connector is None:
+ continue
+
+ try:
+ connector_roles = await connector.get_current_user_group_roles()
+ except NotImplementedError:
+ continue
+ except Exception as e:
+ logger.warning(
+ "Connector group ACL lookup failed",
+ connection_id=getattr(connection, "connection_id", None),
+ connector_type=getattr(connection, "connector_type", None),
+ error=str(e),
+ )
+ continue
+
+ for role in connector_roles or []:
+ role = str(role).strip()
+ if role and role not in seen:
+ seen.add(role)
+ roles.append(role)
+
+ return roles
diff --git a/src/services/knowledge_filter_service.py b/src/services/knowledge_filter_service.py
index 065e97159..a8c7637b1 100644
--- a/src/services/knowledge_filter_service.py
+++ b/src/services/knowledge_filter_service.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any
KNOWLEDGE_FILTERS_INDEX_NAME = "knowledge_filters"
@@ -7,15 +7,25 @@ class KnowledgeFilterService:
def __init__(self, session_manager=None):
self.session_manager = session_manager
+ def _user_client(self, user_id: str = None, jwt_token: str = None):
+ return self.session_manager.get_user_opensearch_client(user_id, jwt_token)
+
+ def _write_client(self, user_id: str = None, jwt_token: str = None):
+ # OpenSearch rejects write requests on indices protected by filter-level
+ # DLS. The app enforces ownership/visibility with the scoped client, then
+ # performs trusted writes with the admin client.
+ from config.settings import clients
+
+ if clients.opensearch is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
+ return clients.opensearch
+
async def create_knowledge_filter(
- self, filter_doc: Dict[str, Any], user_id: str = None, jwt_token: str = None
- ) -> Dict[str, Any]:
+ self, filter_doc: dict[str, Any], user_id: str = None, jwt_token: str = None
+ ) -> dict[str, Any]:
"""Create a new knowledge filter"""
try:
- # Get user's OpenSearch client with JWT for OIDC auth
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
+ opensearch_client = self._write_client(user_id, jwt_token)
# Index the knowledge filter document
result = await opensearch_client.index(
@@ -40,13 +50,10 @@ class KnowledgeFilterService:
async def search_knowledge_filters(
self, query: str, user_id: str = None, jwt_token: str = None, limit: int = 20
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""Search for knowledge filters by name, description, or query content"""
try:
- # Get user's OpenSearch client with JWT for OIDC auth
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
+ opensearch_client = self._user_client(user_id, jwt_token)
if query.strip():
# Search across name, description, and query_data fields
@@ -109,17 +116,12 @@ class KnowledgeFilterService:
async def get_knowledge_filter(
self, filter_id: str, user_id: str = None, jwt_token: str = None
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""Get a specific knowledge filter by ID"""
try:
- # Get user's OpenSearch client with JWT for OIDC auth
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
+ opensearch_client = self._user_client(user_id, jwt_token)
- result = await opensearch_client.get(
- index=KNOWLEDGE_FILTERS_INDEX_NAME, id=filter_id
- )
+ result = await opensearch_client.get(index=KNOWLEDGE_FILTERS_INDEX_NAME, id=filter_id)
if result.get("found"):
knowledge_filter = result["_source"]
@@ -133,16 +135,17 @@ class KnowledgeFilterService:
async def update_knowledge_filter(
self,
filter_id: str,
- updates: Dict[str, Any],
+ updates: dict[str, Any],
user_id: str = None,
jwt_token: str = None,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""Update an existing knowledge filter"""
try:
- # Get user's OpenSearch client with JWT for OIDC auth
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
+ existing = await self.get_knowledge_filter(filter_id, user_id, jwt_token)
+ if not existing.get("success"):
+ return existing
+
+ opensearch_client = self._write_client(user_id, jwt_token)
# Update the document
result = await opensearch_client.update(
@@ -159,10 +162,10 @@ class KnowledgeFilterService:
await opensearch_client.indices.refresh(index=KNOWLEDGE_FILTERS_INDEX_NAME)
except Exception:
pass
- updated_doc = await opensearch_client.get(
- index=KNOWLEDGE_FILTERS_INDEX_NAME, id=filter_id
- )
- return {"success": True, "filter": updated_doc["_source"]}
+ updated_doc = await self.get_knowledge_filter(filter_id, user_id, jwt_token)
+ if updated_doc.get("success"):
+ return updated_doc
+ return {"success": False, "error": "Failed to read updated knowledge filter"}
else:
return {"success": False, "error": "Failed to update knowledge filter"}
@@ -171,13 +174,17 @@ class KnowledgeFilterService:
async def delete_knowledge_filter(
self, filter_id: str, user_id: str = None, jwt_token: str = None
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""Delete a knowledge filter"""
try:
- # Get user's OpenSearch client with JWT for OIDC auth
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
+ existing = await self.get_knowledge_filter(filter_id, user_id, jwt_token)
+ if not existing.get("success"):
+ return {
+ "success": False,
+ "error": "Knowledge filter not found or access denied",
+ }
+
+ opensearch_client = self._write_client(user_id, jwt_token)
result = await opensearch_client.delete(
index=KNOWLEDGE_FILTERS_INDEX_NAME,
@@ -219,20 +226,14 @@ class KnowledgeFilterService:
async def add_subscription(
self,
filter_id: str,
- subscription_data: Dict[str, Any],
+ subscription_data: dict[str, Any],
user_id: str = None,
jwt_token: str = None,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""Add a subscription to a knowledge filter"""
try:
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
-
# Get the current filter document
- filter_result = await self.get_knowledge_filter(
- filter_id, user_id, jwt_token
- )
+ filter_result = await self.get_knowledge_filter(filter_id, user_id, jwt_token)
if not filter_result.get("success"):
return filter_result
@@ -246,12 +247,11 @@ class KnowledgeFilterService:
update_body = {
"doc": {
"subscriptions": subscriptions,
- "updated_at": subscription_data[
- "created_at"
- ], # Use the same timestamp
+ "updated_at": subscription_data["created_at"], # Use the same timestamp
}
}
+ opensearch_client = self._write_client(user_id, jwt_token)
result = await opensearch_client.update(
index=KNOWLEDGE_FILTERS_INDEX_NAME,
id=filter_id,
@@ -273,17 +273,11 @@ class KnowledgeFilterService:
subscription_id: str,
user_id: str = None,
jwt_token: str = None,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""Remove a subscription from a knowledge filter"""
try:
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
-
# Get the current filter document
- filter_result = await self.get_knowledge_filter(
- filter_id, user_id, jwt_token
- )
+ filter_result = await self.get_knowledge_filter(filter_id, user_id, jwt_token)
if not filter_result.get("success"):
return filter_result
@@ -292,9 +286,7 @@ class KnowledgeFilterService:
# Remove subscription from the subscriptions array
subscriptions = filter_doc.get("subscriptions", [])
updated_subscriptions = [
- sub
- for sub in subscriptions
- if sub.get("subscription_id") != subscription_id
+ sub for sub in subscriptions if sub.get("subscription_id") != subscription_id
]
if len(updated_subscriptions) == len(subscriptions):
@@ -310,6 +302,7 @@ class KnowledgeFilterService:
}
}
+ opensearch_client = self._write_client(user_id, jwt_token)
result = await opensearch_client.update(
index=KNOWLEDGE_FILTERS_INDEX_NAME, id=filter_id, body=update_body
)
@@ -324,12 +317,10 @@ class KnowledgeFilterService:
async def get_filter_subscriptions(
self, filter_id: str, user_id: str = None, jwt_token: str = None
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""Get all subscriptions for a knowledge filter"""
try:
- filter_result = await self.get_knowledge_filter(
- filter_id, user_id, jwt_token
- )
+ filter_result = await self.get_knowledge_filter(filter_id, user_id, jwt_token)
if not filter_result.get("success"):
return filter_result
diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.py
index a6b68745c..ae50b0edb 100644
--- a/src/services/langflow_file_service.py
+++ b/src/services/langflow_file_service.py
@@ -1,27 +1,47 @@
import asyncio
+import io
import json
import time
+import uuid
from pathlib import Path
from typing import Any
import httpx
from config.settings import (
+ LANGFLOW_INGEST_CALLBACK_BATCH_SIZE,
LANGFLOW_INGEST_FLOW_ID,
LANGFLOW_URL_INGEST_FLOW_ID,
+ OPENRAG_BACKEND_INTERNAL_URL,
clients,
)
+from services.document_index_writer import DocumentIndexContext
+from utils.hash_utils import hash_id
from utils.logging_config import get_logger
logger = get_logger(__name__)
class LangflowFileService:
- def __init__(self, flows_service=None, docling_service=None):
+ INGEST_OPENSEARCH_COMPONENT_ID = "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-By9U4"
+ URL_INGEST_OPENSEARCH_COMPONENT_ID = (
+ "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV"
+ )
+
+ def __init__(
+ self,
+ flows_service=None,
+ docling_service=None,
+ document_index_writer=None,
+ ingest_token_service=None,
+ ):
self.flow_id_ingest = LANGFLOW_INGEST_FLOW_ID
self.flows_service = flows_service
self.docling_service = docling_service
+ self.document_index_writer = document_index_writer
+ self.ingest_token_service = ingest_token_service
self.flow_id_url_ingest = LANGFLOW_URL_INGEST_FLOW_ID
+ self._embedding_dimension_cache: dict[str, int] = {}
_TRANSIENT_STATUS_CODES = {408, 429, 500, 502, 503, 504}
@@ -71,6 +91,183 @@ class LangflowFileService:
return final_tweaks
+ async def _detect_embedding_dimensions(
+ self,
+ embedding_model: str,
+ embedding_provider: str | None,
+ ) -> int:
+ """Generate one probe embedding so mapping dimensions match the provider."""
+ from services.models_service import ModelsService
+
+ cache_key = f"{embedding_provider or ''}:{embedding_model}"
+ cached = self._embedding_dimension_cache.get(cache_key)
+ if cached:
+ return cached
+
+ litellm_model_name = await ModelsService().get_litellm_model_name(
+ embedding_model,
+ provider=embedding_provider,
+ )
+ response = await clients.patched_embedding_client.embeddings.create(
+ model=litellm_model_name,
+ input=["dimension probe"],
+ )
+ if not response.data:
+ raise RuntimeError("Embedding provider returned no data for dimension probe")
+
+ first = response.data[0]
+ embedding = first["embedding"] if isinstance(first, dict) else first.embedding
+ dimensions = len(embedding)
+ if dimensions <= 0:
+ raise RuntimeError("Embedding provider returned an empty dimension probe")
+
+ self._embedding_dimension_cache[cache_key] = dimensions
+ return dimensions
+
+ async def _ensure_langflow_ingest_index(self, embedding_model: str | None) -> None:
+ """Pre-create index mappings Langflow cannot manage with a DLS JWT."""
+ if clients.opensearch is None:
+ logger.debug("[LF] OpenSearch admin client unavailable; skipping ingest preflight")
+ return
+
+ try:
+ from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL
+ from config.settings import get_index_name, get_openrag_config
+ from utils.embedding_fields import ensure_embedding_field_exists
+ from utils.embeddings import create_index_body
+
+ config = get_openrag_config()
+ index_name = get_index_name()
+ model_name = embedding_model or OPENAI_DEFAULT_EMBEDDING_MODEL
+ embedding_dimensions = await self._detect_embedding_dimensions(
+ model_name,
+ config.knowledge.embedding_provider,
+ )
+ if not await clients.opensearch.indices.exists(index=index_name):
+ await clients.opensearch.indices.create(
+ index=index_name,
+ body=await create_index_body(model_name, embedding_dimensions),
+ )
+
+ await ensure_embedding_field_exists(
+ clients.opensearch,
+ model_name,
+ index_name,
+ embedding_dimensions,
+ )
+ except Exception as e:
+ logger.warning(
+ "[LF] Failed to preconfigure OpenSearch index before Langflow ingest",
+ embedding_model=embedding_model,
+ error=str(e),
+ )
+
+ def _resolve_document_id(
+ self,
+ file_tuples: list[tuple[str, Any, str]] | None,
+ document_id: str | None,
+ ) -> str:
+ if document_id:
+ return document_id
+ if file_tuples and len(file_tuples[0]) > 1:
+ content: Any = file_tuples[0][1]
+ if isinstance(content, str):
+ content = content.encode("utf-8")
+ if isinstance(content, bytes):
+ return hash_id(io.BytesIO(content))
+ return str(uuid.uuid4())
+
+ def _configure_ingest_callback(
+ self,
+ *,
+ document_id: str,
+ filename: str,
+ mimetype: str,
+ file_size: int,
+ embedding_model: str,
+ owner: str | None,
+ owner_name: str | None,
+ owner_email: str | None,
+ connector_type: str | None,
+ source_url: str | None,
+ allowed_users: list[str] | None,
+ allowed_groups: list[str] | None,
+ allowed_principals: list[str] | None,
+ allowed_principal_labels: list[dict[str, Any]] | None = None,
+ ) -> tuple[str | None, str | None]:
+ if self.ingest_token_service is None:
+ return None, None
+
+ from config.settings import get_index_name
+
+ ingest_run_id = f"{document_id}-{uuid.uuid4().hex}"
+ context = DocumentIndexContext(
+ document_id=document_id,
+ filename=filename,
+ mimetype=mimetype,
+ embedding_model=embedding_model,
+ owner=owner,
+ owner_name=owner_name,
+ owner_email=owner_email,
+ file_size=file_size,
+ connector_type=connector_type,
+ source_url=source_url,
+ allowed_users=allowed_users or [],
+ allowed_groups=allowed_groups or [],
+ allowed_principals=allowed_principals or [],
+ allowed_principal_labels=allowed_principal_labels or [],
+ ingest_run_id=ingest_run_id,
+ is_sample_data=connector_type == "openrag_docs",
+ index_name=get_index_name(),
+ )
+ token = self.ingest_token_service.create_token(context)
+ logger.info(
+ "[LF] Configured backend ingest callback",
+ document_id=document_id,
+ ingest_run_id=ingest_run_id,
+ index_name=context.index_name,
+ callback_url=f"{OPENRAG_BACKEND_INTERNAL_URL}/internal/ingest/chunks",
+ )
+ return token, ingest_run_id
+
+ def _ingest_callback_global_var_headers(
+ self,
+ *,
+ ingest_token: str | None,
+ ingest_run_id: str | None,
+ ) -> dict[str, str]:
+ if not ingest_token or not ingest_run_id:
+ return {}
+ return {
+ "X-Langflow-Global-Var-OPENRAG_INGEST_URL": (
+ f"{OPENRAG_BACKEND_INTERNAL_URL}/internal/ingest/chunks"
+ ),
+ "X-Langflow-Global-Var-OPENRAG_INGEST_TOKEN": ingest_token,
+ "X-Langflow-Global-Var-OPENRAG_INGEST_RUN_ID": ingest_run_id,
+ "X-Langflow-Global-Var-OPENRAG_INGEST_BATCH_SIZE": str(
+ LANGFLOW_INGEST_CALLBACK_BATCH_SIZE
+ ),
+ }
+
+ async def _cleanup_failed_callback_ingest(
+ self,
+ *,
+ ingest_token: str | None,
+ ingest_run_id: str | None,
+ ) -> None:
+ if self.ingest_token_service is not None and ingest_token:
+ self.ingest_token_service.revoke_token(ingest_token)
+ if self.document_index_writer is None or not ingest_run_id:
+ return
+ try:
+ await self.document_index_writer.delete_ingest_run(ingest_run_id)
+ except Exception as e:
+ logger.warning(
+ "[LF] Failed to clean up partial backend ingest run",
+ ingest_run_id=ingest_run_id,
+ error=str(e),
+ )
+
async def upload_user_file(self, file_tuple, jwt_token: str | None = None) -> dict[str, Any]:
"""Upload a file using Langflow Files API v2: POST /api/v2/files.
Returns JSON with keys: id, name, path, size, provider.
@@ -131,6 +328,8 @@ class LangflowFileService:
source_url: str | None = None,
allowed_users: list[str] | None = None,
allowed_groups: list[str] | None = None,
+ allowed_principals: list[str] | None = None,
+ allowed_principal_labels: list[dict[str, Any]] | None = None,
selected_embedding_model: str | None = None,
docling_task_id: str | None = None,
) -> dict[str, Any]:
@@ -166,9 +365,6 @@ class LangflowFileService:
metadata_tweaks.append({"key": "connector_type", "value": connector_type})
logger.info(f"[LF] Metadata tweaks {metadata_tweaks}")
- if tweaks:
- payload["tweaks"] = tweaks
- logger.debug(f"[LF] Tweaks {tweaks}")
if session_id:
payload["session_id"] = session_id
@@ -191,6 +387,7 @@ class LangflowFileService:
if file_tuples and len(file_tuples) > 0 and len(file_tuples[0]) > 2
else ""
)
+ resolved_document_id = self._resolve_document_id(file_tuples, document_id)
# Get the current embedding model and provider credentials from config
from config.settings import get_openrag_config
@@ -202,7 +399,7 @@ class LangflowFileService:
embedding_model = selected_embedding_model
headers = {
- "X-Langflow-Global-Var-JWT": str(jwt_token),
+ "X-Langflow-Global-Var-JWT": str(jwt_token or ""),
"X-Langflow-Global-Var-OWNER": str(owner),
"X-Langflow-Global-Var-OWNER_NAME": str(owner_name),
"X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email),
@@ -211,7 +408,7 @@ class LangflowFileService:
"X-Langflow-Global-Var-MIMETYPE": mimetype,
"X-Langflow-Global-Var-FILESIZE": str(file_size_bytes),
"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),
- "X-Langflow-Global-Var-DOCUMENT_ID": str(document_id) if document_id else "",
+ "X-Langflow-Global-Var-DOCUMENT_ID": resolved_document_id,
"X-Langflow-Global-Var-SOURCE_URL": str(source_url) if source_url else "",
"X-Langflow-Global-Var-DOCLING_TASK_ID": str(docling_task_id)
if docling_task_id
@@ -220,15 +417,45 @@ class LangflowFileService:
# Serialize ACL lists as JSON strings for Langflow global vars
# (flows will parse these back into lists before indexing)
- if allowed_users is not None:
- headers["X-Langflow-Global-Var-ALLOWED_USERS"] = json.dumps(allowed_users or [])
- if allowed_groups is not None:
- headers["X-Langflow-Global-Var-ALLOWED_GROUPS"] = json.dumps(allowed_groups or [])
+ headers["X-Langflow-Global-Var-ALLOWED_USERS"] = json.dumps(allowed_users or [])
+ headers["X-Langflow-Global-Var-ALLOWED_GROUPS"] = json.dumps(allowed_groups or [])
+ headers["X-Langflow-Global-Var-ALLOWED_PRINCIPALS"] = json.dumps(allowed_principals or [])
+ headers["X-Langflow-Global-Var-ALLOWED_PRINCIPAL_LABELS"] = json.dumps(
+ allowed_principal_labels or []
+ )
+
+ ingest_token, ingest_run_id = self._configure_ingest_callback(
+ document_id=resolved_document_id,
+ filename=filename,
+ mimetype=mimetype,
+ file_size=file_size_bytes,
+ embedding_model=embedding_model,
+ owner=owner,
+ owner_name=owner_name,
+ owner_email=owner_email,
+ connector_type=connector_type,
+ source_url=source_url,
+ allowed_users=allowed_users,
+ allowed_groups=allowed_groups,
+ allowed_principals=allowed_principals,
+ allowed_principal_labels=allowed_principal_labels,
+ )
+ headers.update(
+ self._ingest_callback_global_var_headers(
+ ingest_token=ingest_token,
+ ingest_run_id=ingest_run_id,
+ )
+ )
+ if tweaks:
+ payload["tweaks"] = tweaks
+ logger.debug("[LF] Tweaks configured", tweak_keys=list(tweaks.keys()))
# Add provider credentials as global variables for ingestion
await add_provider_credentials_to_headers(
headers, config, flows_service=self.flows_service, jwt_token=jwt_token
)
+ if self.ingest_token_service is None:
+ await self._ensure_langflow_ingest_index(embedding_model)
start_time = time.time()
logger.info(
"[INGEST] Run started",
@@ -236,75 +463,82 @@ class LangflowFileService:
filename=filename,
mimetype=mimetype,
)
- resp = await clients.langflow_request(
- "POST",
- f"/api/v1/run/{self.flow_id_ingest}",
- json=payload,
- headers=headers,
- )
- duration = round(time.time() - start_time, 2)
- logger.info(
- "[INGEST] Run complete",
- status_code=resp.status_code,
- reason=resp.reason_phrase,
- duration_s=duration,
- )
- if resp.status_code >= 400:
- logger.error(
- "[LF] Run failed",
+ try:
+ resp = await clients.langflow_request(
+ "POST",
+ f"/api/v1/run/{self.flow_id_ingest}",
+ json=payload,
+ headers=headers,
+ )
+ duration = round(time.time() - start_time, 2)
+ logger.info(
+ "[INGEST] Run complete",
status_code=resp.status_code,
reason=resp.reason_phrase,
- body=resp.text[:1000],
+ duration_s=duration,
)
+ if resp.status_code >= 400:
+ logger.error(
+ "[LF] Run failed",
+ status_code=resp.status_code,
+ reason=resp.reason_phrase,
+ body=resp.text[:1000],
+ )
- # Extract error message from Langflow response
- error_message = f"Server error '{resp.status_code} {resp.reason_phrase}'"
- try:
- error_data = resp.json()
- if isinstance(error_data, dict) and "detail" in error_data:
- detail = error_data["detail"]
- if isinstance(detail, str):
- try:
- detail_obj = json.loads(detail)
- if isinstance(detail_obj, dict) and "message" in detail_obj:
- error_message = detail_obj["message"]
- else:
+ # Extract error message from Langflow response
+ error_message = f"Server error '{resp.status_code} {resp.reason_phrase}'"
+ try:
+ error_data = resp.json()
+ if isinstance(error_data, dict) and "detail" in error_data:
+ detail = error_data["detail"]
+ if isinstance(detail, str):
+ try:
+ detail_obj = json.loads(detail)
+ if isinstance(detail_obj, dict) and "message" in detail_obj:
+ error_message = detail_obj["message"]
+ else:
+ error_message = detail
+ except json.JSONDecodeError:
error_message = detail
- except json.JSONDecodeError:
- error_message = detail
- elif isinstance(detail, dict) and "message" in detail:
- error_message = detail["message"]
- except Exception:
- pass
-
- raise Exception(error_message)
-
- # Check if response is actually JSON before parsing
- content_type = resp.headers.get("content-type", "")
- if "application/json" not in content_type:
- logger.error(
- "[LF] Unexpected response content type from Langflow",
- content_type=content_type,
- status_code=resp.status_code,
- body=resp.text[:1000],
- )
- raise ValueError(
- f"Langflow returned {content_type} instead of JSON. "
- f"This may indicate the ingestion flow failed or the endpoint is incorrect. "
- f"Response preview: {resp.text[:500]}"
- )
+ elif isinstance(detail, dict) and "message" in detail:
+ error_message = detail["message"]
+ except Exception:
+ pass
- try:
- resp_json = resp.json()
- except Exception as e:
- logger.error(
- "[LF] Failed to parse run response as JSON",
- body=resp.text[:1000],
- error=str(e),
- )
+ raise Exception(error_message)
+ # Check if response is actually JSON before parsing
+ content_type = resp.headers.get("content-type", "")
+ if "application/json" not in content_type:
+ logger.error(
+ "[LF] Unexpected response content type from Langflow",
+ content_type=content_type,
+ status_code=resp.status_code,
+ body=resp.text[:1000],
+ )
+ raise ValueError(
+ f"Langflow returned {content_type} instead of JSON. "
+ f"This may indicate the ingestion flow failed or the endpoint is incorrect. "
+ f"Response preview: {resp.text[:500]}"
+ )
+
+ try:
+ resp_json = resp.json()
+ except Exception as e:
+ logger.error(
+ "[LF] Failed to parse run response as JSON",
+ body=resp.text[:1000],
+ error=str(e),
+ )
+
+ raise
+ return resp_json
+ except Exception:
+ await self._cleanup_failed_callback_ingest(
+ ingest_token=ingest_token,
+ ingest_run_id=ingest_run_id,
+ )
raise
- return resp_json
async def run_url_ingestion_flow(
self,
@@ -328,30 +562,61 @@ class LangflowFileService:
"input_type": "chat",
"output_type": "text",
}
- if tweaks:
- payload["tweaks"] = tweaks
+ if not tweaks:
+ tweaks = {}
from config.settings import get_openrag_config
from utils.langflow_headers import add_provider_credentials_to_headers
config = get_openrag_config()
embedding_model = config.knowledge.embedding_model
+ resolved_document_id = hash_id(io.BytesIO(docs_url.encode("utf-8")))
headers = {
- "X-Langflow-Global-Var-JWT": str(jwt_token),
+ "X-Langflow-Global-Var-JWT": str(jwt_token or ""),
"X-Langflow-Global-Var-OWNER": str(owner),
"X-Langflow-Global-Var-OWNER_NAME": str(owner_name),
"X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email),
"X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),
"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),
- "X-Langflow-Global-Var-DOCUMENT_ID": "",
+ "X-Langflow-Global-Var-DOCUMENT_ID": resolved_document_id,
"X-Langflow-Global-Var-SOURCE_URL": str(docs_url),
"X-Langflow-Global-Var-ALLOWED_USERS": json.dumps([]),
"X-Langflow-Global-Var-ALLOWED_GROUPS": json.dumps([]),
+ "X-Langflow-Global-Var-ALLOWED_PRINCIPALS": json.dumps([]),
"X-Langflow-Global-Var-DOCLING_TASK_ID": "",
+ "X-Langflow-Global-Var-FILENAME": str(docs_url),
+ "X-Langflow-Global-Var-MIMETYPE": "text/html",
+ "X-Langflow-Global-Var-FILESIZE": "0",
}
+ ingest_token, ingest_run_id = self._configure_ingest_callback(
+ document_id=resolved_document_id,
+ filename=docs_url,
+ mimetype="text/html",
+ file_size=0,
+ embedding_model=embedding_model,
+ owner=owner,
+ owner_name=owner_name,
+ owner_email=owner_email,
+ connector_type=connector_type,
+ source_url=docs_url,
+ allowed_users=[],
+ allowed_groups=[],
+ allowed_principals=[],
+ allowed_principal_labels=[],
+ )
+ headers.update(
+ self._ingest_callback_global_var_headers(
+ ingest_token=ingest_token,
+ ingest_run_id=ingest_run_id,
+ )
+ )
+ if tweaks:
+ payload["tweaks"] = tweaks
await add_provider_credentials_to_headers(
headers, config, flows_service=self.flows_service, jwt_token=jwt_token
)
+ if self.ingest_token_service is None:
+ await self._ensure_langflow_ingest_index(embedding_model)
logger.info(
"[LF] Running URL ingestion flow",
@@ -359,42 +624,49 @@ class LangflowFileService:
crawl_depth=crawl_depth,
connector_type=connector_type,
embedding_model=embedding_model,
- payload=payload,
- )
- resp = await clients.langflow_request(
- "POST",
- f"/api/v1/run/{flow_id}",
- json=payload,
- headers=headers,
- )
- logger.info(
- "[LF] URL ingestion flow response received",
- status_code=resp.status_code,
- flow_id=flow_id,
+ tweak_keys=list(tweaks.keys()),
)
- if resp.status_code >= 400:
- logger.error(
- "[LF] URL ingestion flow failed",
- status_code=resp.status_code,
- reason=resp.reason_phrase,
- body=resp.text[:1000],
+ try:
+ resp = await clients.langflow_request(
+ "POST",
+ f"/api/v1/run/{flow_id}",
+ json=payload,
+ headers=headers,
)
- resp.raise_for_status()
-
- content_type = resp.headers.get("content-type", "")
- if "application/json" not in content_type:
- logger.error(
- "[LF] Unexpected URL ingestion response content type",
- content_type=content_type,
+ logger.info(
+ "[LF] URL ingestion flow response received",
status_code=resp.status_code,
- body=resp.text[:1000],
- )
- raise ValueError(
- f"Langflow returned {content_type} instead of JSON for URL ingestion. "
- f"Response preview: {resp.text[:500]}"
+ flow_id=flow_id,
)
+ if resp.status_code >= 400:
+ logger.error(
+ "[LF] URL ingestion flow failed",
+ status_code=resp.status_code,
+ reason=resp.reason_phrase,
+ body=resp.text[:1000],
+ )
+ resp.raise_for_status()
- return resp.json()
+ content_type = resp.headers.get("content-type", "")
+ if "application/json" not in content_type:
+ logger.error(
+ "[LF] Unexpected URL ingestion response content type",
+ content_type=content_type,
+ status_code=resp.status_code,
+ body=resp.text[:1000],
+ )
+ raise ValueError(
+ f"Langflow returned {content_type} instead of JSON for URL ingestion. "
+ f"Response preview: {resp.text[:500]}"
+ )
+
+ return resp.json()
+ except Exception:
+ await self._cleanup_failed_callback_ingest(
+ ingest_token=ingest_token,
+ ingest_run_id=ingest_run_id,
+ )
+ raise
async def _ensure_url_ingest_flow_id(self) -> str:
"""Ensure URL ingest flow ID is valid; import flow if missing.
@@ -566,6 +838,8 @@ class LangflowFileService:
source_url: str | None = None,
allowed_users: list[str] | None = None,
allowed_groups: list[str] | None = None,
+ allowed_principals: list[str] | None = None,
+ allowed_principal_labels: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""
Two-phase Docling upload + Langflow ingest operation.
@@ -687,6 +961,8 @@ class LangflowFileService:
source_url=source_url,
allowed_users=allowed_users,
allowed_groups=allowed_groups,
+ allowed_principals=allowed_principals,
+ allowed_principal_labels=allowed_principal_labels,
)
total_duration = round(time.time() - total_start_time, 2)
logger.info(f"[LF] Ingestion completed successfully in {total_duration}s")
diff --git a/src/services/langflow_ingest_token_service.py b/src/services/langflow_ingest_token_service.py
new file mode 100644
index 000000000..5e4d9e79c
--- /dev/null
+++ b/src/services/langflow_ingest_token_service.py
@@ -0,0 +1,153 @@
+"""Short-lived tokens for Langflow-to-backend ingest callbacks."""
+
+from __future__ import annotations
+
+import time
+import uuid
+from typing import Any
+
+import jwt
+from cachetools import TTLCache
+
+from config.settings import LANGFLOW_INGEST_CALLBACK_TTL_SECONDS, SESSION_SECRET
+from services.document_index_writer import DocumentIndexContext
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class LangflowIngestTokenService:
+ """Mint and validate per-run ingest callback tokens.
+
+ A token is valid for one Langflow ingest run. It can be used for multiple
+ batch callbacks, then is marked finalized when the component posts the final
+ batch.
+ """
+
+ audience = "openrag-langflow-ingest"
+ algorithm = "HS256"
+
+ def __init__(self, secret: str | None = None, ttl_seconds: int | None = None):
+ self.secret = secret or SESSION_SECRET
+ self.ttl_seconds = max(ttl_seconds or LANGFLOW_INGEST_CALLBACK_TTL_SECONDS, 1)
+ self._finalized_jtis: TTLCache[str, bool] = TTLCache(
+ maxsize=8192,
+ ttl=self.ttl_seconds + 60,
+ )
+ self._revoked_jtis: TTLCache[str, bool] = TTLCache(
+ maxsize=8192,
+ ttl=self.ttl_seconds + 60,
+ )
+
+ def create_token(self, context: DocumentIndexContext) -> str:
+ now = int(time.time())
+ jti = str(uuid.uuid4())
+ payload = {
+ "aud": self.audience,
+ "scope": "ingest:chunks",
+ "jti": jti,
+ "iat": now,
+ "exp": now + self.ttl_seconds,
+ "ctx": self._context_to_payload(context),
+ }
+ return jwt.encode(payload, self.secret, algorithm=self.algorithm)
+
+ def validate_token(self, token: str) -> tuple[DocumentIndexContext, str]:
+ try:
+ payload = jwt.decode(
+ token,
+ self.secret,
+ algorithms=[self.algorithm],
+ audience=self.audience,
+ )
+ except jwt.PyJWTError as e:
+ logger.warning(
+ "Invalid Langflow ingest token",
+ jwt_error=e.__class__.__name__,
+ detail=str(e),
+ )
+ raise ValueError("Invalid Langflow ingest token") from e
+
+ if payload.get("scope") != "ingest:chunks":
+ raise ValueError("Langflow ingest token has invalid scope")
+
+ jti = str(payload.get("jti") or "")
+ if not jti:
+ raise ValueError("Langflow ingest token is missing jti")
+ if self._revoked_jtis.get(jti) or self._finalized_jtis.get(jti):
+ raise ValueError("Langflow ingest token has already been consumed")
+
+ ctx_payload = payload.get("ctx")
+ if not isinstance(ctx_payload, dict):
+ raise ValueError("Langflow ingest token is missing context")
+
+ return self._payload_to_context(ctx_payload), jti
+
+ def mark_finalized(self, jti: str) -> None:
+ if jti:
+ self._finalized_jtis[jti] = True
+
+ def revoke_token(self, token: str) -> None:
+ try:
+ payload = jwt.decode(
+ token,
+ self.secret,
+ algorithms=[self.algorithm],
+ audience=self.audience,
+ options={"verify_exp": False},
+ )
+ except jwt.PyJWTError:
+ return
+ jti = str(payload.get("jti") or "")
+ if jti:
+ self._revoked_jtis[jti] = True
+
+ @staticmethod
+ def _context_to_payload(context: DocumentIndexContext) -> dict[str, Any]:
+ return {
+ "document_id": context.document_id,
+ "filename": context.filename,
+ "mimetype": context.mimetype,
+ "embedding_model": context.embedding_model,
+ "owner": context.owner,
+ "owner_name": context.owner_name,
+ "owner_email": context.owner_email,
+ "file_size": context.file_size,
+ "connector_type": context.connector_type,
+ "source_url": context.source_url,
+ "allowed_users": list(context.allowed_users),
+ "allowed_groups": list(context.allowed_groups),
+ "allowed_principals": list(context.allowed_principals),
+ "allowed_principal_labels": list(context.allowed_principal_labels),
+ "ingest_run_id": context.ingest_run_id,
+ "is_sample_data": context.is_sample_data,
+ "index_name": context.index_name,
+ }
+
+ @staticmethod
+ def _payload_to_context(payload: dict[str, Any]) -> DocumentIndexContext:
+ file_size = payload.get("file_size")
+ if file_size is not None and not isinstance(file_size, int):
+ try:
+ file_size = int(file_size)
+ except (TypeError, ValueError):
+ file_size = None
+ return DocumentIndexContext(
+ document_id=str(payload.get("document_id") or ""),
+ filename=str(payload.get("filename") or ""),
+ mimetype=str(payload.get("mimetype") or ""),
+ embedding_model=str(payload.get("embedding_model") or ""),
+ owner=payload.get("owner"),
+ owner_name=payload.get("owner_name"),
+ owner_email=payload.get("owner_email"),
+ file_size=file_size,
+ connector_type=payload.get("connector_type"),
+ source_url=payload.get("source_url"),
+ allowed_users=list(payload.get("allowed_users") or []),
+ allowed_groups=list(payload.get("allowed_groups") or []),
+ allowed_principals=list(payload.get("allowed_principals") or []),
+ allowed_principal_labels=list(payload.get("allowed_principal_labels") or []),
+ ingest_run_id=payload.get("ingest_run_id"),
+ is_sample_data=bool(payload.get("is_sample_data")),
+ index_name=payload.get("index_name"),
+ )
diff --git a/src/services/monitor_service.py b/src/services/monitor_service.py
index 7c78c6f0a..dd33e9dc1 100644
--- a/src/services/monitor_service.py
+++ b/src/services/monitor_service.py
@@ -1,7 +1,7 @@
-import uuid
import json
-from typing import Any, Dict, Optional, List
-from datetime import datetime
+import uuid
+from typing import Any
+
from config.settings import clients
from utils.logging_config import get_logger
@@ -13,23 +13,38 @@ class MonitorService:
self.session_manager = session_manager
self.webhook_base_url = webhook_base_url or "http://openrag-backend:8000"
+ def _trusted_opensearch_client(self):
+ if clients.opensearch is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
+ return clients.opensearch
+
+ @staticmethod
+ def _monitor_has_user_tag(monitor: dict[str, Any], user_id: str) -> bool:
+ user_tag = f"user:{user_id}"
+ inputs = monitor.get("_source", monitor).get("monitor", {}).get("inputs", [])
+ for monitor_input in inputs:
+ for query in monitor_input.get("doc_level_input", {}).get("queries", []):
+ if user_tag in (query.get("tags") or []):
+ return True
+ return False
+
async def create_knowledge_filter_monitor(
self,
filter_id: str,
filter_name: str,
- query_data: Dict[str, Any],
+ query_data: dict[str, Any],
user_id: str,
jwt_token: str,
- notification_config: Dict[str, Any] = None,
- ) -> Dict[str, Any]:
+ notification_config: dict[str, Any] = None,
+ ) -> dict[str, Any]:
"""Create a document-level monitor for a knowledge filter"""
try:
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
+ opensearch_client = self._trusted_opensearch_client()
subscription_id = str(uuid.uuid4())
- webhook_url = f"{self.webhook_base_url}/knowledge-filter/{filter_id}/webhook/{subscription_id}"
+ webhook_url = (
+ f"{self.webhook_base_url}/knowledge-filter/{filter_id}/webhook/{subscription_id}"
+ )
# Convert knowledge filter query to monitor query format
monitor_query = self._convert_kf_query_to_monitor_query(query_data)
@@ -70,9 +85,7 @@ class MonitorService:
"document_level_trigger": {
"name": f"KF Trigger: {filter_name}",
"severity": "1",
- "condition": {
- "script": {"source": "return true", "lang": "painless"}
- },
+ "condition": {"script": {"source": "return true", "lang": "painless"}},
"actions": [
{
"name": f"KF Webhook Action: {filter_name}",
@@ -122,14 +135,15 @@ class MonitorService:
except Exception as e:
return {"success": False, "error": f"Monitor creation failed: {str(e)}"}
- async def delete_monitor(
- self, monitor_id: str, user_id: str, jwt_token: str
- ) -> Dict[str, Any]:
+ async def delete_monitor(self, monitor_id: str, user_id: str, jwt_token: str) -> dict[str, Any]:
"""Delete a document-level monitor"""
try:
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
+ opensearch_client = self._trusted_opensearch_client()
+ existing = await opensearch_client.transport.perform_request(
+ "GET", f"/_plugins/_alerting/monitors/{monitor_id}"
)
+ if not self._monitor_has_user_tag(existing, user_id):
+ return {"success": False, "error": "Not authorized to delete this monitor"}
response = await opensearch_client.transport.perform_request(
"DELETE", f"/_plugins/_alerting/monitors/{monitor_id}"
@@ -143,20 +157,18 @@ class MonitorService:
except Exception as e:
return {"success": False, "error": f"Monitor deletion failed: {str(e)}"}
- async def get_monitor(
- self, monitor_id: str, user_id: str, jwt_token: str
- ) -> Dict[str, Any]:
+ async def get_monitor(self, monitor_id: str, user_id: str, jwt_token: str) -> dict[str, Any]:
"""Get monitor details"""
try:
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
+ opensearch_client = self._trusted_opensearch_client()
response = await opensearch_client.transport.perform_request(
"GET", f"/_plugins/_alerting/monitors/{monitor_id}"
)
if response.get("_id"):
+ if not self._monitor_has_user_tag(response, user_id):
+ return {"success": False, "error": "Monitor not found"}
return {"success": True, "monitor": response}
else:
return {"success": False, "error": "Monitor not found"}
@@ -166,17 +178,25 @@ class MonitorService:
async def list_user_monitors(
self, user_id: str, jwt_token: str, limit: int = 50
- ) -> List[Dict[str, Any]]:
+ ) -> list[dict[str, Any]]:
"""List all monitors for a specific user"""
try:
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
+ opensearch_client = self._trusted_opensearch_client()
- # Search for all monitors (DLS will filter to user's monitors automatically)
+ # Search with an explicit user tag because the trusted backend
+ # client bypasses the user's OpenSearch DLS filter.
search_body = {
"query": {
- "bool": {"must": [{"term": {"monitor.type": "doc_level_monitor"}}]}
+ "bool": {
+ "must": [
+ {"term": {"monitor.type": "doc_level_monitor"}},
+ {
+ "term": {
+ "monitor.inputs.doc_level_input.queries.tags": f"user:{user_id}"
+ }
+ },
+ ]
+ }
},
"sort": [{"monitor.last_update_time": {"order": "desc"}}],
"size": limit,
@@ -195,19 +215,15 @@ class MonitorService:
return monitors
except Exception as e:
- logger.error(
- "Error listing monitors for user", user_id=user_id, error=str(e)
- )
+ logger.error("Error listing monitors for user", user_id=user_id, error=str(e))
return []
async def list_monitors_for_filter(
self, filter_id: str, user_id: str, jwt_token: str
- ) -> List[Dict[str, Any]]:
+ ) -> list[dict[str, Any]]:
"""List all monitors for a specific knowledge filter"""
try:
- opensearch_client = self.session_manager.get_user_opensearch_client(
- user_id, jwt_token
- )
+ opensearch_client = self._trusted_opensearch_client()
# Search for monitors with the knowledge filter tag
search_body = {
@@ -220,6 +236,11 @@ class MonitorService:
"monitor.inputs.doc_level_input.queries.tags": f"knowledge_filter:{filter_id}"
}
},
+ {
+ "term": {
+ "monitor.inputs.doc_level_input.queries.tags": f"user:{user_id}"
+ }
+ },
]
}
}
@@ -238,14 +259,10 @@ class MonitorService:
return monitors
except Exception as e:
- logger.error(
- "Error listing monitors for filter", filter_id=filter_id, error=str(e)
- )
+ logger.error("Error listing monitors for filter", filter_id=filter_id, error=str(e))
return []
- async def _get_or_create_webhook_destination(
- self, webhook_url: str, opensearch_client
- ) -> str:
+ async def _get_or_create_webhook_destination(self, webhook_url: str, opensearch_client) -> str:
"""Get or create a webhook destination for notifications"""
try:
# Try to find existing webhook destination
@@ -257,10 +274,7 @@ class MonitorService:
# Check if we already have a destination for this webhook URL
for config in search_response.get("config_list", []):
- if (
- config.get("config", {}).get("webhook", {}).get("url")
- == webhook_url
- ):
+ if config.get("config", {}).get("webhook", {}).get("url") == webhook_url:
return config["config_id"]
# Create new webhook destination
@@ -285,11 +299,9 @@ class MonitorService:
return response.get("config_id")
except Exception as e:
- raise Exception(f"Failed to create webhook destination: {str(e)}")
+ raise RuntimeError(f"Failed to create webhook destination: {str(e)}") from e
- def _convert_kf_query_to_monitor_query(
- self, query_data: Dict[str, Any]
- ) -> Dict[str, Any]:
+ def _convert_kf_query_to_monitor_query(self, query_data: dict[str, Any]) -> dict[str, Any]:
"""Convert knowledge filter query format to OpenSearch monitor query format"""
# This assumes the query_data contains an OpenSearch query structure
# You may need to adjust this based on your actual knowledge filter query format
diff --git a/src/services/search_service.py b/src/services/search_service.py
index b4a0f8935..f9a1d7702 100644
--- a/src/services/search_service.py
+++ b/src/services/search_service.py
@@ -1,14 +1,15 @@
import asyncio
import copy
import os
-import json
from collections import Counter
-from typing import Any, Dict
+from typing import Any
+
from agentd.tool_decorator import tool
-from config.settings import clients, get_embedding_model, get_index_name, get_openrag_config
+
+from auth_context import get_auth_context
from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL
+from config.settings import clients, get_embedding_model, get_index_name, get_openrag_config
from utils.container_utils import transform_localhost_url
-from auth_context import get_auth_context
from utils.logging_config import get_logger
logger = get_logger(__name__)
@@ -32,7 +33,7 @@ def register_search_service(service: "SearchService") -> None:
@tool
-async def search_tool(query: str, embedding_model: str = None) -> Dict[str, Any]:
+async def search_tool(query: str, embedding_model: str = None) -> dict[str, Any]:
"""
Use this tool to search for documents relevant to the query.
@@ -70,7 +71,7 @@ class SearchService:
except Exception as e:
logger.warning("[SEARCH] Could not configure Ollama endpoint from config", error=str(e))
- async def search_tool(self, query: str, embedding_model: str = None) -> Dict[str, Any]:
+ async def search_tool(self, query: str, embedding_model: str = None) -> dict[str, Any]:
"""
Use this tool to search for documents relevant to the query.
@@ -102,9 +103,9 @@ class SearchService:
user_id, jwt_token = get_auth_context()
# Get search filters, limit, and score threshold from context
from auth_context import (
+ get_score_threshold,
get_search_filters,
get_search_limit,
- get_score_threshold,
)
filters = get_search_filters() or {}
@@ -122,7 +123,7 @@ class SearchService:
if not is_wildcard_match_all:
# Build filter clauses first so we can use them in model detection
- filter_clauses = []
+ filter_clauses: list[dict[str, Any]] = []
if filters:
# Map frontend filter names to backend field names
field_mapping = {
@@ -249,7 +250,7 @@ class SearchService:
return_exceptions=True,
)
- for model_name, result in zip(available_models, embedding_results):
+ for model_name, result in zip(available_models, embedding_results, strict=False):
if isinstance(result, BaseException):
failed_models.append(model_name)
logger.warning(
@@ -299,7 +300,7 @@ class SearchService:
if is_wildcard_match_all:
# Match all documents; still allow filters to narrow scope
if filter_clauses:
- query_block = {"bool": {"filter": filter_clauses}}
+ query_block: dict[str, Any] = {"bool": {"filter": filter_clauses}}
else:
query_block = {"match_all": {}}
else:
@@ -328,7 +329,9 @@ class SearchService:
# fallback mode.
all_filters = list(filter_clauses)
if knn_queries:
- exists_should = [{"exists": {"field": f}} for f in embedding_fields_to_check]
+ exists_should: list[dict[str, Any]] = [
+ {"exists": {"field": f}} for f in embedding_fields_to_check
+ ]
# Docs indexed under a failed provider have none of the successful
# embedding fields, but keyword matching should still surface them.
# Allow them through by matching on their embedding_model value.
@@ -402,7 +405,7 @@ class SearchService:
}
}
- search_body = {
+ search_body: dict[str, Any] = {
"query": query_block,
"aggs": {
"data_sources": {"terms": {"field": "filename", "size": 20}},
@@ -426,6 +429,7 @@ class SearchService:
"embedding_dimensions",
"allowed_users",
"allowed_groups",
+ "allowed_principal_labels",
],
"size": limit,
}
@@ -436,7 +440,7 @@ class SearchService:
# Prepare fallback search body without num_candidates for clusters that don't support it.
# Only relevant when we actually dispatched KNN queries.
- fallback_search_body = None
+ fallback_search_body: dict[str, Any] | None = None
if not is_wildcard_match_all and query_embeddings:
try:
fallback_search_body = copy.deepcopy(search_body)
@@ -466,10 +470,11 @@ class SearchService:
opensearch_client = self.session_manager.get_user_opensearch_client(user_id, jwt_token)
from opensearchpy.exceptions import RequestError
+
from utils.opensearch_utils import (
+ DISK_SPACE_ERROR_MESSAGE,
OpenSearchDiskSpaceError,
is_disk_space_error,
- DISK_SPACE_ERROR_MESSAGE,
)
search_params = {"terminate_after": 0}
@@ -554,6 +559,7 @@ class SearchService:
# ACL fields (may be missing for some documents)
"allowed_users": source.get("allowed_users", []),
"allowed_groups": source.get("allowed_groups", []),
+ "allowed_principal_labels": source.get("allowed_principal_labels", []),
}
)
@@ -578,7 +584,7 @@ class SearchService:
if exact_files:
chunks = [chunk for chunk in chunks if chunk.get("filename") in exact_files]
- def _build_terms_agg(field: str) -> Dict[str, Any]:
+ def _build_terms_agg(field: str) -> dict[str, Any]:
counts = Counter(
value
for chunk in chunks
@@ -606,7 +612,7 @@ class SearchService:
# Return both transformed results and aggregations. Surface degraded
# semantic-search signals so the UI can show a non-fatal warning
# instead of treating partial-embedding failure as a hard error.
- response: Dict[str, Any] = {
+ response: dict[str, Any] = {
"results": chunks,
"aggregations": aggregations,
"total": len(chunks),
@@ -633,11 +639,11 @@ class SearchService:
query: str,
user_id: str = None,
jwt_token: str = None,
- filters: Dict[str, Any] = None,
+ filters: dict[str, Any] = None,
limit: int = 10,
score_threshold: float = 0,
embedding_model: str = None,
- ) -> Dict[str, Any]:
+ ) -> dict[str, Any]:
"""Public search method for API endpoints
Args:
@@ -658,7 +664,7 @@ class SearchService:
set_search_filters(filters)
- from auth_context import set_search_limit, set_score_threshold
+ from auth_context import set_score_threshold, set_search_limit
set_search_limit(limit)
set_score_threshold(score_threshold)
diff --git a/src/services/user_service.py b/src/services/user_service.py
index c8e8b8611..b15ed0c1d 100644
--- a/src/services/user_service.py
+++ b/src/services/user_service.py
@@ -68,6 +68,8 @@ async def ensure_user_row(
await user_repo.update_last_login(existing.id)
if jwt_roles is not None:
await _sync_jwt_roles(role_repo, audit_repo, existing.id, jwt_roles)
+ else:
+ await _assign_default_role_if_missing(session, role_repo, audit_repo, existing.id)
return existing
# Possible legacy row (oauth_provider='legacy', oauth_subject==user_id)
@@ -295,6 +297,18 @@ async def _assign_default_role(
)
+async def _assign_default_role_if_missing(
+ session: AsyncSession,
+ role_repo: RoleRepo,
+ audit_repo: AuditRepo,
+ user_id: str,
+) -> None:
+ roles = await role_repo.list_user_roles(user_id)
+ if roles:
+ return
+ await _assign_default_role(session, role_repo, audit_repo, user_id)
+
+
async def get_effective_provider_keys(session: AsyncSession, user_id: str) -> dict:
"""Phase-4 helper. Returns a dict of provider -> overrides for this user.
diff --git a/src/session_manager.py b/src/session_manager.py
index faf553e25..81e5c80ae 100644
--- a/src/session_manager.py
+++ b/src/session_manager.py
@@ -1,3 +1,4 @@
+import hashlib
import os
import time
from dataclasses import dataclass
@@ -128,7 +129,7 @@ class SessionManager:
self.algorithm = "HS256"
else:
if IBM_AUTH_ENABLED:
- # IBM Auth Mode: Traefik handles auth (no local JWT signing required)
+ # External auth handles authentication; local signing is not required.
self.private_key = None
self.public_key = None
self.public_key_pem = None
@@ -148,7 +149,8 @@ class SessionManager:
with open(self.public_key_path, "rb") as f:
self.public_key = serialization.load_pem_public_key(f.read())
- self.public_key_pem = open(self.public_key_path).read()
+ with open(self.public_key_path) as f:
+ self.public_key_pem = f.read()
except FileNotFoundError as e:
raise Exception(f"RSA key files not found: {e}") from e
@@ -203,22 +205,30 @@ class SessionManager:
# Create JWT token using the shared method
return self.create_jwt_token(user)
- def create_jwt_token(self, user: User) -> str:
- """Create JWT token for an existing user"""
+ def _get_oidc_issuer(self) -> str:
# Use OpenSearch-compatible issuer for OIDC validation
+ from config.settings import OPENRAG_FQDN
+
oidc_issuer = "http://openrag-backend:8000"
- openrag_fqdn = os.getenv("OPENRAG_FQDN")
- if openrag_fqdn:
- oidc_issuer = f"http://{openrag_fqdn}:8000"
+ if OPENRAG_FQDN:
+ oidc_issuer = f"http://{OPENRAG_FQDN}:8000"
+ return oidc_issuer
+ def _create_signed_jwt_token(
+ self,
+ user: User,
+ *,
+ expires_delta: timedelta,
+ ) -> str:
# Create JWT token with OIDC-compliant claims
now = datetime.utcnow()
+ roles = ["openrag_user"]
token_payload = {
# OIDC standard claims
- "iss": oidc_issuer, # Fixed issuer for OpenSearch OIDC
+ "iss": self._get_oidc_issuer(), # Fixed issuer for OpenSearch OIDC
"sub": user.user_id, # Subject (user ID)
"aud": ["opensearch", "openrag"], # Audience
- "exp": now + timedelta(days=7), # Expiration
+ "exp": now + expires_delta, # Expiration
"iat": now, # Issued at
"auth_time": int(now.timestamp()), # Authentication time
# Custom claims
@@ -227,21 +237,38 @@ class SessionManager:
"name": user.name,
"preferred_username": user.email,
"email_verified": True,
- "roles": ["openrag_user"], # Backend role for OpenSearch
- "user_roles": ["openrag_user", "all_access"], # compatible with OpenSearch's roles_key
+ "roles": roles, # Backend roles for OpenSearch
+ "user_roles": roles + ["all_access"], # compatible with OpenSearch's roles_key
}
- # Check for token from environment variable first
- token = os.getenv("OPENSEARCH_JWT_TOKEN")
- if token and (token.startswith("Bearer ") or token.startswith("Basic ")):
- return token
- if not token:
- if self.private_key is None:
- logger.error("create_jwt_token called but JWT signing is disabled (IBM auth mode)")
- return None
- token = jwt.encode(token_payload, self.private_key, algorithm=self.algorithm)
+ if self.private_key is None:
+ logger.error("JWT signing requested but signing is disabled")
+ return None
+ token = jwt.encode(token_payload, self.private_key, algorithm=self.algorithm)
return f"Bearer {token}"
+ def create_jwt_token(self, user: User) -> str:
+ """Create the long-lived OpenRAG session JWT for an existing user."""
+ return self._create_signed_jwt_token(
+ user,
+ expires_delta=timedelta(days=7),
+ )
+
+ def create_opensearch_jwt_token(
+ self,
+ user: User,
+ ttl_seconds: int | None = None,
+ ) -> str:
+ """Create a short-lived OpenSearch JWT for user-scoped OpenSearch requests."""
+ if ttl_seconds is None:
+ from config.settings import get_opensearch_jwt_ttl_seconds
+
+ ttl_seconds = get_opensearch_jwt_ttl_seconds()
+ return self._create_signed_jwt_token(
+ user,
+ expires_delta=timedelta(seconds=max(ttl_seconds, 1)),
+ )
+
def verify_token(self, token: str) -> dict[str, Any] | None:
"""Verify JWT token and return decoded claims, using an in-process cache."""
if IBM_AUTH_ENABLED:
@@ -290,26 +317,35 @@ class SessionManager:
else:
user_id = user_or_id
+ provided_jwt_token = jwt_token
+
# Get the effective JWT token (handles anonymous JWT creation)
jwt_token = self.get_effective_jwt_token(user_id, jwt_token)
from config.settings import clients
- # In IBM mode credentials may rotate per-request — always create a fresh client
+ # Upstream credentials may rotate per request.
if IBM_AUTH_ENABLED:
return clients.create_user_opensearch_client(jwt_token)
- # Check if we have a cached client for this user
- if user_id not in self.user_opensearch_clients:
- self.user_opensearch_clients[user_id] = clients.create_user_opensearch_client(jwt_token)
+ if provided_jwt_token:
+ token_hash = hashlib.sha256(jwt_token.encode("utf-8")).hexdigest()
+ cache_key = f"{user_id}:{token_hash}"
+ else:
+ cache_key = user_id or "anonymous"
+
+ if cache_key not in self.user_opensearch_clients:
+ self.user_opensearch_clients[cache_key] = clients.create_user_opensearch_client(
+ jwt_token
+ )
- return self.user_opensearch_clients[user_id]
+ return self.user_opensearch_clients[cache_key]
def get_effective_jwt_token(self, user_id: str, jwt_token: str) -> str:
"""Get the effective JWT token, creating anonymous JWT if needed in no-auth mode"""
from config.settings import is_no_auth_mode
- # IBM JWT is used as-is — never override with an anonymous OpenRAG JWT
+ # Upstream JWT is used as-is; never override it with an anonymous token.
if IBM_AUTH_ENABLED and jwt_token:
return jwt_token
@@ -318,7 +354,7 @@ class SessionManager:
# No token — create one
if is_no_auth_mode() or user_id in (None, AnonymousUser().user_id):
- # IBM Auth Mode: No anonymous JWT concept (disable signing)
+ # No anonymous JWT concept when local signing is disabled.
if self.private_key is None:
return None
# anonymous JWT (cached)
diff --git a/src/utils/acl_utils.py b/src/utils/acl_utils.py
index 13d79e79c..5375dae3f 100644
--- a/src/utils/acl_utils.py
+++ b/src/utils/acl_utils.py
@@ -16,20 +16,20 @@ from utils.logging_config import get_logger
logger = get_logger(__name__)
-def compute_acl_hash(acl: DocumentACL) -> str:
- """
- Compute SHA256 hash of ACL for change detection.
+def _sorted_principal_labels(labels: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
+ return sorted(
+ [label for label in labels or [] if isinstance(label, dict)],
+ key=lambda label: json.dumps(label, sort_keys=True),
+ )
- Args:
- acl: DocumentACL instance
- Returns:
- Hexadecimal hash string
- """
+def compute_acl_hash(acl: DocumentACL) -> str:
+ """Compute SHA256 hash of ACL fields that are refreshed after ingest."""
acl_data = {
- "owner": acl.owner,
"allowed_users": sorted(acl.allowed_users),
"allowed_groups": sorted(acl.allowed_groups),
+ "allowed_principals": sorted(acl.allowed_principals),
+ "allowed_principal_labels": _sorted_principal_labels(acl.allowed_principal_labels),
}
return hashlib.sha256(json.dumps(acl_data, sort_keys=True).encode()).hexdigest()
@@ -58,55 +58,41 @@ async def should_update_acl(
opensearch_client,
id_fields: tuple[str, ...] = ("document_id",),
) -> bool:
- """
- Check if ACL has changed by querying one chunk and comparing hashes.
+ """Return whether indexed ACL lists differ from ``new_acl``.
- This optimization queries only a single chunk instead of updating all chunks,
- enabling efficient skip when ACLs haven't changed (most common case).
-
- Args:
- document_id: Document identifier
- new_acl: New ACL to compare against
- opensearch_client: OpenSearch client instance
-
- Returns:
- True if ACL has changed and update is needed, False otherwise
+ The owner field is intentionally not compared here: ownership is set at
+ ingest to the authenticated uploading/syncing user and ACL refresh must not
+ reassign it to the upstream file author.
"""
try:
- # Query one chunk for this document
response = await opensearch_client.search(
index="documents",
body={
"query": _build_id_query(document_id, id_fields),
"size": 1,
"_source": [
- "owner",
"allowed_users",
"allowed_groups",
+ "allowed_principals",
+ "allowed_principal_labels",
],
},
)
if not response["hits"]["hits"]:
- # New document, need to index
return True
existing_chunk = response["hits"]["hits"][0]["_source"]
-
- # Reconstruct existing ACL and compute hash
existing_acl = DocumentACL(
- owner=existing_chunk.get("owner"),
allowed_users=existing_chunk.get("allowed_users", []),
allowed_groups=existing_chunk.get("allowed_groups", []),
+ allowed_principals=existing_chunk.get("allowed_principals", []),
+ allowed_principal_labels=existing_chunk.get("allowed_principal_labels", []),
)
- existing_hash = compute_acl_hash(existing_acl)
- new_hash = compute_acl_hash(new_acl)
-
- return existing_hash != new_hash
+ return compute_acl_hash(existing_acl) != compute_acl_hash(new_acl)
except Exception as e:
- # On error, assume update needed to be safe
logger.error("[OPENSEARCH] ACL check failed", document_id=document_id, error=str(e))
return True
@@ -115,40 +101,31 @@ async def update_document_acl(
document_id: str,
acl: DocumentACL,
opensearch_client,
+ write_opensearch_client=None,
id_fields: tuple[str, ...] = ("document_id",),
) -> dict[str, Any]:
- """
- Update ACL for all chunks of a document.
-
- Uses hash-based skip optimization: queries one chunk to check if ACL changed,
- only updates if changed. When updating, uses bulk update_by_query for efficiency.
+ """Update ACL lists for all chunks of a document.
- Only the access lists (``allowed_users``/``allowed_groups``) are updated — the
- ``owner`` field is intentionally left untouched so an ACL re-sync never
- reassigns document ownership (owner is set once at ingest to the syncing user).
-
- Args:
- document_id: Document identifier
- acl: New ACL to apply
- opensearch_client: OpenSearch client instance
- id_fields: Chunk fields to match ``document_id`` against. Defaults to
- ``("document_id",)``; pass ``("document_id", "connector_file_id")`` for
- connector chunks where the connector id lives in ``connector_file_id``.
-
- Returns:
- Dict with status ("unchanged" or "updated") and chunks_updated count
+ The user-scoped ``opensearch_client`` is used for visibility/change checks;
+ ``write_opensearch_client`` performs the mutation. Only access-list fields
+ are updated. ``owner`` is intentionally left untouched.
"""
- # Check if ACL changed (queries one chunk)
should_update = await should_update_acl(
- document_id, acl, opensearch_client, id_fields=id_fields
+ document_id,
+ acl,
+ opensearch_client,
+ id_fields=id_fields,
)
if not should_update:
return {"status": "unchanged", "chunks_updated": 0}
- # Bulk update all chunks for this document
+ write_client = write_opensearch_client
+ if write_client is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
+
try:
- response = await opensearch_client.update_by_query(
+ response = await write_client.update_by_query(
index="documents",
body={
"query": _build_id_query(document_id, id_fields),
@@ -156,10 +133,14 @@ async def update_document_acl(
"source": """
ctx._source.allowed_users = params.allowed_users;
ctx._source.allowed_groups = params.allowed_groups;
+ ctx._source.allowed_principals = params.allowed_principals;
+ ctx._source.allowed_principal_labels = params.allowed_principal_labels;
""",
"params": {
"allowed_users": acl.allowed_users,
"allowed_groups": acl.allowed_groups,
+ "allowed_principals": acl.allowed_principals,
+ "allowed_principal_labels": acl.allowed_principal_labels,
},
},
},
@@ -175,39 +156,19 @@ async def update_document_acl(
async def batch_update_acls(
acl_updates: list[tuple[str, DocumentACL]],
opensearch_client,
+ write_opensearch_client=None,
id_fields: tuple[str, ...] = ("document_id",),
) -> dict[str, Any]:
- """
- Batch update ACLs for multiple documents.
-
- Optimizations:
- - Parallel ACL change detection (query one chunk per document)
- - Skip unchanged ACLs (95%+ of webhook notifications)
- - Parallel bulk updates for changed ACLs
-
- Only ``allowed_users``/``allowed_groups`` are updated; ``owner`` is left
- untouched (see :func:`update_document_acl`).
-
- Args:
- acl_updates: List of (document_id, acl) tuples
- opensearch_client: OpenSearch client instance
- id_fields: Chunk fields to match the id against (see
- :func:`update_document_acl`).
-
- Returns:
- Dict with status, documents_updated count, and chunks_updated count
- """
+ """Batch update ACL lists for multiple documents."""
if not acl_updates:
return {"status": "no_updates", "documents_updated": 0, "chunks_updated": 0}
- # Filter to only changed ACLs (parallel chunk queries)
check_tasks = [
should_update_acl(doc_id, acl, opensearch_client, id_fields=id_fields)
for doc_id, acl in acl_updates
]
should_update_flags = await asyncio.gather(*check_tasks)
- # Filter to documents with changed ACLs
changed = [
(doc_id, acl)
for (doc_id, acl), should_update in zip(acl_updates, should_update_flags, strict=True)
@@ -222,9 +183,12 @@ async def batch_update_acls(
"skipped": len(acl_updates),
}
- # Bulk update chunks for each document (parallelized)
+ write_client = write_opensearch_client
+ if write_client is None:
+ raise RuntimeError("Backend OpenSearch write client is unavailable")
+
update_tasks = [
- opensearch_client.update_by_query(
+ write_client.update_by_query(
index="documents",
body={
"query": _build_id_query(doc_id, id_fields),
@@ -232,10 +196,14 @@ async def batch_update_acls(
"source": """
ctx._source.allowed_users = params.allowed_users;
ctx._source.allowed_groups = params.allowed_groups;
+ ctx._source.allowed_principals = params.allowed_principals;
+ ctx._source.allowed_principal_labels = params.allowed_principal_labels;
""",
"params": {
"allowed_users": acl.allowed_users,
"allowed_groups": acl.allowed_groups,
+ "allowed_principals": acl.allowed_principals,
+ "allowed_principal_labels": acl.allowed_principal_labels,
},
},
},
@@ -245,8 +213,6 @@ async def batch_update_acls(
try:
results = await asyncio.gather(*update_tasks, return_exceptions=True)
-
- # Count successful updates
total_chunks_updated = 0
errors = []
for result in results:
diff --git a/src/utils/embeddings.py b/src/utils/embeddings.py
index 1ee591eb0..4226324d8 100644
--- a/src/utils/embeddings.py
+++ b/src/utils/embeddings.py
@@ -1,39 +1,64 @@
+from utils.embedding_fields import build_knn_vector_field, get_embedding_field_name
from utils.logging_config import get_logger
logger = get_logger(__name__)
-async def create_index_body() -> dict:
+
+async def create_index_body(
+ embedding_model: str | None = None,
+ embedding_dimensions: int | None = None,
+) -> dict:
"""Create a static index body configuration.
-
+
Returns:
OpenSearch index body configuration
"""
+ from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL
+ from config.settings import (
+ ACL_PRINCIPAL_LABELS_MAPPING,
+ VECTOR_DIM,
+ get_openrag_config,
+ )
+
+ resolved_embedding_model = (
+ embedding_model
+ or get_openrag_config().knowledge.embedding_model
+ or OPENAI_DEFAULT_EMBEDDING_MODEL
+ )
+
+ properties = {
+ "document_id": {"type": "keyword"},
+ "filename": {"type": "keyword"},
+ "mimetype": {"type": "keyword"},
+ "page": {"type": "integer"},
+ "text": {"type": "text"},
+ # Legacy field - kept for backward compatibility and for clusters where
+ # Langflow cannot perform mapping updates with a DLS-filtered JWT.
+ "chunk_embedding": build_knn_vector_field(VECTOR_DIM),
+ # Track which embedding model was used for this chunk
+ "embedding_model": {"type": "keyword"},
+ "embedding_dimensions": {"type": "integer"},
+ "source_url": {"type": "keyword"},
+ "connector_type": {"type": "keyword"},
+ "ingest_run_id": {"type": "keyword"},
+ "owner": {"type": "keyword"},
+ "owner_email": {"type": "keyword"},
+ "allowed_users": {"type": "keyword"},
+ "allowed_groups": {"type": "keyword"},
+ "allowed_principals": {"type": "keyword"},
+ "allowed_principal_labels": ACL_PRINCIPAL_LABELS_MAPPING,
+ "created_time": {"type": "date"},
+ "modified_time": {"type": "date"},
+ "indexed_time": {"type": "date"},
+ "metadata": {"type": "object"},
+ }
+
+ if embedding_dimensions:
+ properties[get_embedding_field_name(resolved_embedding_model)] = build_knn_vector_field(
+ embedding_dimensions
+ )
return {
- "settings": {
- "index": {"knn": True},
- "number_of_shards": 1,
- "number_of_replicas": 0
- },
- "mappings": {
- "properties": {
- "document_id": {"type": "keyword"},
- "filename": {"type": "keyword"},
- "mimetype": {"type": "keyword"},
- "page": {"type": "integer"},
- "text": {"type": "text"},
- # Track which embedding model was used for this chunk
- "embedding_model": {"type": "keyword"},
- "embedding_dimensions": {"type": "integer"},
- "source_url": {"type": "keyword"},
- "connector_type": {"type": "keyword"},
- "owner": {"type": "keyword"},
- "allowed_users": {"type": "keyword"},
- "allowed_groups": {"type": "keyword"},
- "created_time": {"type": "date"},
- "modified_time": {"type": "date"},
- "indexed_time": {"type": "date"},
- "metadata": {"type": "object"},
- }
- },
+ "settings": {"index": {"knn": True}, "number_of_shards": 1, "number_of_replicas": 0},
+ "mappings": {"properties": properties},
}
diff --git a/src/utils/group_acl.py b/src/utils/group_acl.py
new file mode 100644
index 000000000..1954cfdde
--- /dev/null
+++ b/src/utils/group_acl.py
@@ -0,0 +1,161 @@
+"""Helpers for compact connector group ACL role names."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import re
+import uuid
+from collections.abc import Iterable
+from typing import Any
+
+_SAFE_COMPONENT_RE = re.compile(r"^[a-z0-9_-]+$")
+
+
+def compact_acl_component(value: object, *, max_length: int = 48) -> str:
+ """Return a short, role-safe component for provider/tenant/group IDs."""
+ text = str(value or "").strip().lower()
+ if not text:
+ raise ValueError("ACL role component cannot be empty")
+
+ try:
+ parsed_uuid = uuid.UUID(text)
+ except ValueError:
+ if _SAFE_COMPONENT_RE.fullmatch(text) and len(text) <= max_length:
+ return text
+ digest = hashlib.sha256(text.encode("utf-8")).digest()
+ return "h" + base64.urlsafe_b64encode(digest[:16]).rstrip(b"=").decode("ascii")
+
+ return base64.urlsafe_b64encode(parsed_uuid.bytes).rstrip(b"=").decode("ascii")
+
+
+def canonical_group_role(provider_code: str, tenant_id: object, group_id: object) -> str:
+ """Build the OpenSearch backend role used for connector group ACLs."""
+ provider = compact_acl_component(provider_code, max_length=16)
+ tenant = compact_acl_component(tenant_id or "global")
+ group = compact_acl_component(group_id)
+ return f"g:{provider}:{tenant}:{group}"
+
+
+def canonical_user_principal(provider_code: str, tenant_id: object, user_id: object) -> str:
+ """Build the provider-scoped principal used for connector user ACLs."""
+ provider = compact_acl_component(provider_code, max_length=16)
+ tenant = compact_acl_component(tenant_id or "global")
+ user = compact_acl_component(user_id)
+ return f"u:{provider}:{tenant}:{user}"
+
+
+def canonical_group_roles(
+ provider_code: str,
+ tenant_id: object,
+ group_ids: Iterable[object],
+) -> list[str]:
+ """Canonicalize and deduplicate group IDs while preserving first-seen order."""
+ roles: list[str] = []
+ seen: set[str] = set()
+ for group_id in group_ids or ():
+ try:
+ role = canonical_group_role(provider_code, tenant_id, group_id)
+ except ValueError:
+ continue
+ if role not in seen:
+ seen.add(role)
+ roles.append(role)
+ return roles
+
+
+def canonical_user_principals(
+ provider_code: str,
+ tenant_id: object,
+ user_ids: Iterable[object],
+) -> list[str]:
+ """Canonicalize and deduplicate user IDs while preserving first-seen order."""
+ principals: list[str] = []
+ seen: set[str] = set()
+ for user_id in user_ids or ():
+ try:
+ principal = canonical_user_principal(provider_code, tenant_id, user_id)
+ except ValueError:
+ continue
+ if principal not in seen:
+ seen.add(principal)
+ principals.append(principal)
+ return principals
+
+
+def unique_acl_principals(principals: Iterable[object]) -> list[str]:
+ """Return non-empty ACL principals without duplicates, preserving order."""
+ unique: list[str] = []
+ seen: set[str] = set()
+ for principal in principals or ():
+ value = str(principal or "").strip()
+ if value and value not in seen:
+ seen.add(value)
+ unique.append(value)
+ return unique
+
+
+def acl_principal_label(
+ principal: object,
+ *,
+ kind: str,
+ provider: str,
+ display_name: object = None,
+ email: object = None,
+ external_id: object = None,
+) -> dict[str, Any] | None:
+ """Build non-authoritative display metadata for an ACL principal."""
+ principal_value = str(principal or "").strip()
+ if not principal_value:
+ return None
+
+ label: dict[str, Any] = {
+ "principal": principal_value,
+ "kind": str(kind or "").strip() or "unknown",
+ "provider": str(provider or "").strip() or "unknown",
+ }
+ for key, value in (
+ ("display_name", display_name),
+ ("email", email),
+ ("external_id", external_id),
+ ):
+ if value is None:
+ continue
+ value_text = str(value).strip()
+ if value_text:
+ label[key] = value_text
+ return label
+
+
+def unique_acl_principal_labels(labels: Iterable[object]) -> list[dict[str, Any]]:
+ """Return display labels keyed by canonical principal, preserving first-seen order."""
+ unique: list[dict[str, Any]] = []
+ by_principal: dict[str, dict[str, Any]] = {}
+
+ for raw_label in labels or ():
+ if not isinstance(raw_label, dict):
+ continue
+ principal = str(raw_label.get("principal") or "").strip()
+ if not principal:
+ continue
+
+ label: dict[str, Any] = {"principal": principal}
+ for key in ("kind", "provider", "display_name", "email", "external_id"):
+ value = raw_label.get(key)
+ if value is None:
+ continue
+ value_text = str(value).strip()
+ if value_text:
+ label[key] = value_text
+
+ existing = by_principal.get(principal)
+ if existing is None:
+ by_principal[principal] = label
+ unique.append(label)
+ continue
+
+ for key, value in label.items():
+ if key not in existing and value:
+ existing[key] = value
+
+ return unique
diff --git a/src/utils/opensearch_init.py b/src/utils/opensearch_init.py
index f15991029..f365e1f5e 100644
--- a/src/utils/opensearch_init.py
+++ b/src/utils/opensearch_init.py
@@ -6,8 +6,11 @@ that need to (re)create the documents index after onboarding completes.
"""
from config.settings import (
+ ACL_PRINCIPAL_LABELS_MAPPING,
API_KEYS_INDEX_BODY,
API_KEYS_INDEX_NAME,
+ DLS_PRINCIPAL_INDEX_BODY,
+ DLS_PRINCIPAL_INDEX_NAME,
IBM_AUTH_ENABLED,
INDEX_BODY,
OPENRAG_SKIP_OS_SECURITY_SETUP,
@@ -23,6 +26,82 @@ from utils.telemetry import Category, MessageId, TelemetryClient
logger = get_logger(__name__)
+async def _ensure_keyword_mappings(os_client, index_name: str, field_names: list[str]) -> None:
+ """Add missing keyword mappings to an existing index.
+
+ New ACL fields must be explicit keyword fields before documents are indexed;
+ otherwise OpenSearch dynamic mapping can create analyzed text fields that do
+ not work for exact-match DLS terms queries.
+ """
+ try:
+ current_mapping = await os_client.indices.get_mapping(index=index_name)
+ properties = current_mapping.get(index_name, {}).get("mappings", {}).get("properties", {})
+ missing: dict[str, dict[str, str]] = {}
+ for field_name in field_names:
+ existing = properties.get(field_name)
+ if existing is None:
+ missing[field_name] = {"type": "keyword"}
+ elif existing.get("type") != "keyword":
+ logger.warning(
+ "OpenSearch field has incompatible mapping for DLS exact match",
+ index_name=index_name,
+ field_name=field_name,
+ mapping=existing,
+ )
+
+ if missing:
+ await os_client.indices.put_mapping(
+ index=index_name,
+ body={"properties": missing},
+ )
+ logger.info(
+ "Updated OpenSearch keyword mappings",
+ index_name=index_name,
+ fields=list(missing),
+ )
+ except Exception as e:
+ logger.warning(
+ "Failed to ensure OpenSearch keyword mappings",
+ index_name=index_name,
+ fields=field_names,
+ error=str(e),
+ )
+
+
+async def _ensure_field_mappings(
+ os_client,
+ index_name: str,
+ field_mappings: dict[str, dict],
+) -> None:
+ """Add missing explicit mappings to an existing index."""
+ try:
+ current_mapping = await os_client.indices.get_mapping(index=index_name)
+ properties = current_mapping.get(index_name, {}).get("mappings", {}).get("properties", {})
+ missing = {
+ field_name: mapping
+ for field_name, mapping in field_mappings.items()
+ if properties.get(field_name) is None
+ }
+
+ if missing:
+ await os_client.indices.put_mapping(
+ index=index_name,
+ body={"properties": missing},
+ )
+ logger.info(
+ "Updated OpenSearch field mappings",
+ index_name=index_name,
+ fields=list(missing),
+ )
+ except Exception as e:
+ logger.warning(
+ "Failed to ensure OpenSearch field mappings",
+ index_name=index_name,
+ fields=list(field_mappings),
+ error=str(e),
+ )
+
+
async def wait_for_opensearch(opensearch_client=None):
"""Wait for OpenSearch to be ready, delegating to the shared utility."""
from utils.opensearch_utils import (
@@ -73,6 +152,16 @@ async def _ensure_opensearch_index():
index_name = get_index_name()
if await clients.opensearch.indices.exists(index=index_name):
logger.info("[OPENSEARCH] Index already exists", index_name=index_name)
+ await _ensure_keyword_mappings(
+ clients.opensearch,
+ index_name,
+ ["allowed_users", "allowed_groups", "allowed_principals", "ingest_run_id"],
+ )
+ await _ensure_field_mappings(
+ clients.opensearch,
+ index_name,
+ {"allowed_principal_labels": ACL_PRINCIPAL_LABELS_MAPPING},
+ )
return
await clients.opensearch.indices.create(index=index_name, body=INDEX_BODY)
@@ -137,6 +226,16 @@ async def init_index(opensearch_client=None, admin_username: str = None):
index_name=index_name,
embedding_model=embedding_model,
)
+ await _ensure_keyword_mappings(
+ os_client,
+ index_name,
+ ["allowed_users", "allowed_groups", "allowed_principals", "ingest_run_id"],
+ )
+ await _ensure_field_mappings(
+ os_client,
+ index_name,
+ {"allowed_principal_labels": ACL_PRINCIPAL_LABELS_MAPPING},
+ )
if not (IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE):
# Set number of replicas to 0 to not create unused nodes in OpenSearch, in case it was created with more replicas
try:
@@ -174,6 +273,7 @@ async def init_index(opensearch_client=None, admin_username: str = None):
"owner": {"type": "keyword"},
"allowed_users": {"type": "keyword"},
"allowed_groups": {"type": "keyword"},
+ "allowed_principals": {"type": "keyword"},
"subscriptions": {"type": "object"},
"created_at": {"type": "date"},
"updated_at": {"type": "date"},
@@ -197,6 +297,11 @@ async def init_index(opensearch_client=None, admin_username: str = None):
"Knowledge filters index already exists, skipping creation",
index_name=knowledge_filter_index_name,
)
+ await _ensure_keyword_mappings(
+ os_client,
+ knowledge_filter_index_name,
+ ["allowed_users", "allowed_groups", "allowed_principals"],
+ )
if not (IBM_AUTH_ENABLED and PLATFORM_AUTH_DEV_MODE):
try:
@@ -230,6 +335,28 @@ async def init_index(opensearch_client=None, admin_username: str = None):
index_name=API_KEYS_INDEX_NAME,
)
+ if not await os_client.indices.exists(index=DLS_PRINCIPAL_INDEX_NAME):
+ await os_client.indices.create(
+ index=DLS_PRINCIPAL_INDEX_NAME,
+ body=DLS_PRINCIPAL_INDEX_BODY,
+ )
+ logger.info("Created DLS principal index", index_name=DLS_PRINCIPAL_INDEX_NAME)
+ else:
+ logger.info(
+ "DLS principal index already exists, skipping creation",
+ index_name=DLS_PRINCIPAL_INDEX_NAME,
+ )
+ await _ensure_keyword_mappings(
+ os_client,
+ DLS_PRINCIPAL_INDEX_NAME,
+ ["user_name", "auth_user_id", "auth_email", "provider", "principals"],
+ )
+ await _ensure_field_mappings(
+ os_client,
+ DLS_PRINCIPAL_INDEX_NAME,
+ {"principal_labels": ACL_PRINCIPAL_LABELS_MAPPING},
+ )
+
await configure_alerting_security()
except Exception as e:
diff --git a/src/utils/opensearch_queries.py b/src/utils/opensearch_queries.py
index eadafd376..a99d3227f 100644
--- a/src/utils/opensearch_queries.py
+++ b/src/utils/opensearch_queries.py
@@ -1,7 +1,6 @@
"""
Utility functions for constructing OpenSearch queries consistently.
"""
-from typing import Union, List
def build_filename_query(filename: str) -> dict:
@@ -14,14 +13,12 @@ def build_filename_query(filename: str) -> dict:
Returns:
A dict containing the OpenSearch query body
"""
- return {
- "term": {
- "filename": filename
- }
- }
+ return {"term": {"filename": filename}}
-def build_filename_search_body(filename: str, size: int = 1, source: Union[bool, List[str]] = False) -> dict:
+def build_filename_search_body(
+ filename: str, size: int = 1, source: bool | list[str] = False
+) -> dict:
"""
Build a complete search body for checking if a filename exists.
@@ -33,23 +30,16 @@ def build_filename_search_body(filename: str, size: int = 1, source: Union[bool,
Returns:
A dict containing the complete OpenSearch search body
"""
- return {
- "query": build_filename_query(filename),
- "size": size,
- "_source": source
- }
-
+ return {"query": build_filename_query(filename), "size": size, "_source": source}
-def build_filename_delete_body(filename: str) -> dict:
- """
- Build a delete-by-query body for removing all documents with a filename.
- Args:
- filename: The exact filename to delete
-
- Returns:
- A dict containing the OpenSearch delete-by-query body
- """
+def build_owned_filename_query(filename: str, owner: str) -> dict:
+ """Build a query for chunks with a filename owned by a specific user."""
return {
- "query": build_filename_query(filename)
- }
\ No newline at end of file
+ "bool": {
+ "filter": [
+ build_filename_query(filename),
+ {"term": {"owner": owner}},
+ ]
+ }
+ }
diff --git a/tests/conftest.py b/tests/conftest.py
index f76b85770..03adae5ac 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -12,8 +12,8 @@ load_dotenv()
# Force no-auth mode for testing by setting OAuth credentials to empty strings
# This ensures anonymous JWT tokens are created automatically
-os.environ['GOOGLE_OAUTH_CLIENT_ID'] = ''
-os.environ['GOOGLE_OAUTH_CLIENT_SECRET'] = ''
+os.environ["GOOGLE_OAUTH_CLIENT_ID"] = ""
+os.environ["GOOGLE_OAUTH_CLIENT_SECRET"] = ""
# RBAC is OFF by default in production. For tests we keep it ON so the
# RBAC assertions in admin-endpoint and require_permission tests aren't
@@ -31,13 +31,13 @@ if not os.environ.get("SDK_TESTS_ONLY") == "true":
_test_db_path = Path(_test_db_dir) / "openrag.db"
os.environ["DATABASE_URL"] = f"sqlite+aiosqlite:///{_test_db_path}"
-from config.settings import clients
-from session_manager import SessionManager
-from main import generate_jwt_keys
+from config.settings import clients # noqa: E402
+from main import generate_jwt_keys # noqa: E402
+from session_manager import SessionManager # noqa: E402
@pytest_asyncio.fixture(scope="session", autouse=True)
-async def onboard_system():
+async def onboard_system(request):
"""Perform initial onboarding once for all tests in the session.
This ensures the OpenRAG config is marked as edited and properly initialized
@@ -49,6 +49,12 @@ async def onboard_system():
if os.environ.get("SDK_TESTS_ONLY") == "true":
yield
return
+ selected_items = getattr(request.session, "items", [])
+ if selected_items and all(
+ item.get_closest_marker("openrag_skip_app_onboard") for item in selected_items
+ ):
+ yield
+ return
# Delete any existing config to ensure clean onboarding
config_file = Path("config/config.yaml")
@@ -63,11 +69,13 @@ async def onboard_system():
# require_permission-gated endpoint dies with
# "no such table: permissions".
from db.migrations_runtime import run_alembic_upgrade_async
+
await run_alembic_upgrade_async("head")
# Bind the SQLAlchemy engine to the migrated DB. init_engine is
# idempotent and synchronous; safe to call from the test event loop.
from db.engine import init_engine
+
init_engine()
# Clean up OpenSearch indices to ensure fresh state for tests
@@ -78,11 +86,11 @@ async def onboard_system():
except Exception as e:
print(f"[DEBUG] Could not wipe OpenSearch indices: {e}")
-
# Create app and perform onboarding via API
- from main import create_app, startup_tasks
import httpx
+ from main import create_app, startup_tasks
+
app = await create_app()
await startup_tasks(app.state.services)
@@ -99,7 +107,7 @@ async def onboard_system():
# If it fails, it might already be onboarded, which is fine
print(f"[DEBUG] Onboarding returned {resp.status_code}: {resp.text}")
else:
- print(f"[DEBUG] Session onboarding completed successfully")
+ print("[DEBUG] Session onboarding completed successfully")
yield
@@ -136,7 +144,9 @@ def session_manager():
# Generate RSA keys before creating SessionManager
generate_jwt_keys()
sm = SessionManager("test-secret-key")
- print(f"[DEBUG] SessionManager created with keys: private={sm.private_key_path}, public={sm.public_key_path}")
+ print(
+ f"[DEBUG] SessionManager created with keys: private={sm.private_key_path}, public={sm.public_key_path}"
+ )
return sm
@@ -145,31 +155,41 @@ def test_documents_dir():
"""Create a temporary directory with test documents."""
with tempfile.TemporaryDirectory() as temp_dir:
test_dir = Path(temp_dir)
-
+
# Create some test files in supported formats
- (test_dir / "test1.md").write_text("# Machine Learning Document\n\nThis is a test document about machine learning.")
- (test_dir / "test2.md").write_text("# AI Document\n\nAnother document discussing artificial intelligence.")
- (test_dir / "test3.md").write_text("# Data Science Document\n\nThis is a markdown file about data science.")
-
+ (test_dir / "test1.md").write_text(
+ "# Machine Learning Document\n\nThis is a test document about machine learning."
+ )
+ (test_dir / "test2.md").write_text(
+ "# AI Document\n\nAnother document discussing artificial intelligence."
+ )
+ (test_dir / "test3.md").write_text(
+ "# Data Science Document\n\nThis is a markdown file about data science."
+ )
+
# Create subdirectory with files
sub_dir = test_dir / "subdir"
sub_dir.mkdir()
- (sub_dir / "nested.md").write_text("# Neural Networks\n\nNested document about neural networks.")
-
+ (sub_dir / "nested.md").write_text(
+ "# Neural Networks\n\nNested document about neural networks."
+ )
+
yield test_dir
@pytest.fixture
def test_single_file():
"""Create a single test file."""
- with tempfile.NamedTemporaryFile(mode='w', suffix='_test_document.md', delete=False) as f:
- f.write("# Single Test Document\n\nThis is a test document about OpenRAG testing framework. This document contains multiple sentences to ensure proper chunking. The content should be indexed and searchable in OpenSearch after processing.")
+ with tempfile.NamedTemporaryFile(mode="w", suffix="_test_document.md", delete=False) as f:
+ f.write(
+ "# Single Test Document\n\nThis is a test document about OpenRAG testing framework. This document contains multiple sentences to ensure proper chunking. The content should be indexed and searchable in OpenSearch after processing."
+ )
temp_path = f.name
-
+
yield temp_path
-
+
# Cleanup
try:
os.unlink(temp_path)
except FileNotFoundError:
- pass
\ No newline at end of file
+ pass
diff --git a/tests/integration/core/test_api_endpoints.py b/tests/integration/core/test_api_endpoints.py
index 9444739d7..a7019aa62 100644
--- a/tests/integration/core/test_api_endpoints.py
+++ b/tests/integration/core/test_api_endpoints.py
@@ -165,8 +165,11 @@ async def test_upload_and_search_endpoint(tmp_path: Path, disable_langflow_inges
"services.startup_orchestrator",
"app.routes.internal",
"app.routes",
+ "app.container",
"app.factory",
"app.lifespan",
+ "dependencies",
+ "utils.opensearch_init",
]:
sys.modules.pop(mod, None)
from config.settings import clients, get_index_name
@@ -390,8 +393,11 @@ async def test_langflow_chat_and_nudges_endpoints():
"services.startup_orchestrator",
"app.routes.internal",
"app.routes",
+ "app.container",
"app.factory",
"services.chat_service",
+ "dependencies",
+ "utils.opensearch_init",
]:
sys.modules.pop(mod, None)
@@ -497,8 +503,11 @@ async def test_search_multi_embedding_models(tmp_path: Path):
"services.startup_orchestrator",
"app.routes.internal",
"app.routes",
+ "app.container",
"app.factory",
"app.lifespan",
+ "dependencies",
+ "utils.opensearch_init",
]:
sys.modules.pop(mod, None)
@@ -641,8 +650,11 @@ async def test_router_upload_ingest_traditional(tmp_path: Path, disable_langflow
"services.startup_orchestrator",
"app.routes.internal",
"app.routes",
+ "app.container",
"app.factory",
"app.lifespan",
+ "dependencies",
+ "utils.opensearch_init",
]:
sys.modules.pop(mod, None)
from config.settings import clients, get_index_name
diff --git a/tests/integration/core/test_group_acl_dls.py b/tests/integration/core/test_group_acl_dls.py
new file mode 100644
index 000000000..3e42da771
--- /dev/null
+++ b/tests/integration/core/test_group_acl_dls.py
@@ -0,0 +1,202 @@
+from uuid import uuid4
+
+import pytest
+from opensearchpy import AsyncOpenSearch
+from opensearchpy._async.http_aiohttp import AIOHttpConnection
+
+pytestmark = [
+ pytest.mark.asyncio,
+ pytest.mark.openrag_skip_app_onboard,
+]
+
+
+def _build_admin_opensearch_client():
+ from config.settings import (
+ IBM_AUTH_ENABLED,
+ OPENSEARCH_HOST,
+ OPENSEARCH_PASSWORD,
+ OPENSEARCH_PORT,
+ OPENSEARCH_USERNAME,
+ )
+
+ if IBM_AUTH_ENABLED:
+ pytest.skip("OSS JWT DLS group matching is not used in IBM auth mode")
+ if not OPENSEARCH_PASSWORD:
+ pytest.skip("OPENSEARCH_PASSWORD is required for direct OpenSearch DLS integration test")
+
+ return AsyncOpenSearch(
+ hosts=[{"host": OPENSEARCH_HOST, "port": OPENSEARCH_PORT}],
+ connection_class=AIOHttpConnection,
+ scheme="https",
+ use_ssl=True,
+ verify_certs=False,
+ ssl_assert_fingerprint=None,
+ http_auth=(OPENSEARCH_USERNAME, OPENSEARCH_PASSWORD),
+ http_compress=True,
+ )
+
+
+async def _search_visible_document_ids(opensearch_client, index_name: str) -> set[str]:
+ response = await opensearch_client.search(
+ index=index_name,
+ body={
+ "query": {"match_all": {}},
+ "sort": [{"document_id": "asc"}],
+ "_source": ["document_id"],
+ "size": 10,
+ },
+ )
+ return {hit["_source"]["document_id"] for hit in response.get("hits", {}).get("hits", [])}
+
+
+async def test_opensearch_dls_filters_group_and_user_acl_documents():
+ """Prove user and connector principal ACLs are enforced by OpenSearch DLS.
+
+ This test avoids live connector dependencies. The admin client seeds a
+ documents* index with ACL-only docs, then a user-scoped OpenSearch client
+ searches with JWT subject/email claims and lookup-table principals that
+ should and should not match. Connector groups are not passed through JWT
+ roles.
+ """
+ from config.settings import (
+ DLS_PRINCIPAL_INDEX_BODY,
+ DLS_PRINCIPAL_INDEX_NAME,
+ INDEX_BODY,
+ clients,
+ )
+ from session_manager import SessionManager, User
+ from utils.opensearch_utils import setup_opensearch_security
+
+ admin_client = _build_admin_opensearch_client()
+ try:
+ is_reachable = await admin_client.ping()
+ except Exception:
+ is_reachable = False
+ if not is_reachable:
+ await admin_client.close()
+ pytest.skip("OpenSearch is not reachable")
+
+ index_name = f"documents_group_acl_dls_{uuid4().hex}"
+ user_id = f"group-dls-user-{uuid4().hex}"
+ user_email = f"{user_id}@example.com"
+ matching_group = "g:test:tenant:engineering"
+ other_group = "g:test:tenant:sales"
+ user_principal = "u:test:tenant:reader"
+ other_principal = "u:test:tenant:other"
+
+ try:
+ await setup_opensearch_security(admin_client)
+
+ await admin_client.indices.create(index=index_name, body=INDEX_BODY)
+ if not await admin_client.indices.exists(index=DLS_PRINCIPAL_INDEX_NAME):
+ await admin_client.indices.create(
+ index=DLS_PRINCIPAL_INDEX_NAME,
+ body=DLS_PRINCIPAL_INDEX_BODY,
+ )
+ await admin_client.index(
+ index=DLS_PRINCIPAL_INDEX_NAME,
+ id=user_id,
+ body={
+ "user_name": user_id,
+ "auth_user_id": user_id,
+ "auth_email": user_email,
+ "provider": "test",
+ "principals": [matching_group, user_principal],
+ "updated_at": "2026-05-15T00:00:00+00:00",
+ },
+ refresh=True,
+ )
+ await admin_client.bulk(
+ body=[
+ {"index": {"_index": index_name, "_id": "engineering-doc"}},
+ {
+ "document_id": "engineering-doc",
+ "filename": "engineering.md",
+ "text": "Visible only to engineering",
+ "owner": "external-owner",
+ "allowed_users": [],
+ "allowed_groups": [matching_group],
+ "allowed_principals": [matching_group],
+ },
+ {"index": {"_index": index_name, "_id": "subject-doc"}},
+ {
+ "document_id": "subject-doc",
+ "filename": "subject.md",
+ "text": "Visible to the matching JWT subject",
+ "owner": "external-owner",
+ "allowed_users": [user_id],
+ "allowed_groups": [],
+ "allowed_principals": [],
+ },
+ {"index": {"_index": index_name, "_id": "email-doc"}},
+ {
+ "document_id": "email-doc",
+ "filename": "email.md",
+ "text": "Visible to the matching JWT email",
+ "owner": "external-owner",
+ "allowed_users": [user_email],
+ "allowed_groups": [],
+ "allowed_principals": [],
+ },
+ {"index": {"_index": index_name, "_id": "principal-user-doc"}},
+ {
+ "document_id": "principal-user-doc",
+ "filename": "principal-user.md",
+ "text": "Visible to the matching DLS user principal",
+ "owner": "external-owner",
+ "allowed_users": [],
+ "allowed_groups": [],
+ "allowed_principals": [user_principal],
+ },
+ {"index": {"_index": index_name, "_id": "principal-group-doc"}},
+ {
+ "document_id": "principal-group-doc",
+ "filename": "principal-group.md",
+ "text": "Visible to the matching DLS group principal lookup",
+ "owner": "external-owner",
+ "allowed_users": [],
+ "allowed_groups": [],
+ "allowed_principals": [matching_group],
+ },
+ {"index": {"_index": index_name, "_id": "sales-doc"}},
+ {
+ "document_id": "sales-doc",
+ "filename": "sales.md",
+ "text": "Visible only to sales",
+ "owner": "external-owner",
+ "allowed_users": [],
+ "allowed_groups": [other_group],
+ "allowed_principals": [other_principal],
+ },
+ ],
+ refresh=True,
+ )
+
+ session_manager = SessionManager("test")
+ user = User(
+ user_id=user_id,
+ email=user_email,
+ name="Group DLS User",
+ )
+ token = session_manager.create_opensearch_jwt_token(user, ttl_seconds=120)
+
+ user_client = clients.create_user_opensearch_client(token)
+ try:
+ assert await _search_visible_document_ids(user_client, index_name) == {
+ "email-doc",
+ "engineering-doc",
+ "principal-group-doc",
+ "principal-user-doc",
+ "subject-doc",
+ }
+ finally:
+ await user_client.close()
+ finally:
+ await admin_client.indices.delete(index=index_name, ignore_unavailable=True)
+ await admin_client.delete(
+ index=DLS_PRINCIPAL_INDEX_NAME,
+ id=user_id,
+ ignore=[404],
+ refresh=True,
+ )
+ await admin_client.close()
diff --git a/tests/integration/core/test_onboarding_sample_docs.py b/tests/integration/core/test_onboarding_sample_docs.py
new file mode 100644
index 000000000..ea68b3812
--- /dev/null
+++ b/tests/integration/core/test_onboarding_sample_docs.py
@@ -0,0 +1,240 @@
+import asyncio
+import os
+import sys
+from pathlib import Path
+from uuid import uuid4
+
+import httpx
+import pytest
+import pytest_asyncio
+
+pytestmark = [
+ pytest.mark.asyncio,
+ pytest.mark.openrag_skip_app_onboard,
+]
+
+
+_RELOAD_MODULES = [
+ "api.router",
+ "api.connector_router",
+ "api.settings.endpoints",
+ "api.settings.helpers",
+ "api.settings.langflow_sync",
+ "api.settings",
+ "api",
+ "app.container",
+ "app.routes.internal",
+ "app.routes.public_v1",
+ "app.routes",
+ "app.factory",
+ "app.lifespan",
+ "auth_middleware",
+ "config.config_manager",
+ "config.settings",
+ "dependencies",
+ "main",
+ "services",
+ "services.default_docs_service",
+ "services.search_service",
+ "services.startup_orchestrator",
+ "utils.opensearch_init",
+]
+
+_EXCLUDED_DEFAULT_DOCS = {"warmup_ocr.pdf"}
+
+
+def _reload_openrag_modules() -> None:
+ for module_name in _RELOAD_MODULES:
+ sys.modules.pop(module_name, None)
+
+
+async def _require_langflow_ready() -> None:
+ langflow_url = os.getenv("LANGFLOW_URL", "http://localhost:7860").rstrip("/")
+ try:
+ async with httpx.AsyncClient(timeout=2.0) as client:
+ response = await client.get(f"{langflow_url}/health")
+ except Exception as exc:
+ pytest.skip(f"Langflow is required for onboarding integration tests: {exc}")
+
+ if response.status_code != 200:
+ pytest.skip(
+ "Langflow is required for onboarding integration tests: "
+ f"{langflow_url}/health returned {response.status_code}"
+ )
+
+
+@pytest_asyncio.fixture
+async def isolated_onboarding_docs_workspace(tmp_path: Path, monkeypatch):
+ if not os.getenv("OPENAI_API_KEY"):
+ pytest.skip("OPENAI_API_KEY is required for onboarding sample-doc ingestion")
+
+ docs_dir = Path(__file__).resolve().parents[3] / "openrag-documents"
+ expected_filenames = sorted(
+ path.name
+ for path in docs_dir.rglob("*")
+ if path.is_file() and path.name not in _EXCLUDED_DEFAULT_DOCS
+ )
+ if not expected_filenames:
+ pytest.fail(f"No default docs found in {docs_dir}")
+
+ config_dir = tmp_path / "config"
+ data_dir = tmp_path / "data"
+ keys_dir = tmp_path / "keys"
+ for directory in (config_dir, data_dir, keys_dir):
+ directory.mkdir()
+
+ index_name = f"documents_onboarding_sample_{uuid4().hex}"
+ db_path = tmp_path / "openrag.db"
+
+ monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{db_path}")
+ monkeypatch.setenv("OPENRAG_CONFIG_PATH", str(config_dir))
+ monkeypatch.setenv("OPENRAG_DATA_PATH", str(data_dir))
+ monkeypatch.setenv("OPENRAG_DOCUMENTS_PATH", str(docs_dir))
+ monkeypatch.setenv("OPENRAG_KEYS_PATH", str(keys_dir))
+ monkeypatch.setenv("OPENSEARCH_INDEX_NAME", index_name)
+ monkeypatch.setenv("INGEST_SAMPLE_DATA", "true")
+ monkeypatch.setenv("DEFAULT_DOCS_INGEST_SOURCE", "files")
+ monkeypatch.setenv("DISABLE_INGEST_WITH_LANGFLOW", "false")
+ monkeypatch.setenv("DISABLE_STARTUP_INGEST", "true")
+ monkeypatch.setenv("FETCH_OPENRAG_DOCS_AT_STARTUP", "false")
+ monkeypatch.setenv("GOOGLE_OAUTH_CLIENT_ID", "")
+ monkeypatch.setenv("GOOGLE_OAUTH_CLIENT_SECRET", "")
+ monkeypatch.setenv("OPENRAG_NOAUTH_ROLE", "admin")
+ monkeypatch.setenv("OPENRAG_RBAC_ENFORCE", "true")
+
+ from db.engine import dispose_engine
+ from dependencies import invalidate_user_ensured_cache
+
+ await dispose_engine()
+ invalidate_user_ensured_cache()
+ _reload_openrag_modules()
+ await _require_langflow_ready()
+
+ try:
+ yield {
+ "index_name": index_name,
+ "expected_filenames": expected_filenames,
+ }
+ finally:
+ try:
+ from config.settings import clients
+
+ if clients.opensearch is not None:
+ await clients.opensearch.indices.delete(
+ index=index_name,
+ ignore_unavailable=True,
+ )
+ await clients.close()
+ except Exception:
+ pass
+ await dispose_engine()
+ invalidate_user_ensured_cache()
+ _reload_openrag_modules()
+
+
+async def _wait_for_task(task_service, task_id: str, timeout_s: float = 900.0) -> dict:
+ deadline = asyncio.get_event_loop().time() + timeout_s
+ last_status = None
+ while asyncio.get_event_loop().time() < deadline:
+ last_status = task_service.get_task_status("anonymous", task_id)
+ if last_status and last_status.get("status") in {"completed", "failed"}:
+ return last_status
+ await asyncio.sleep(0.5)
+ raise AssertionError(f"Sample-doc ingestion task did not finish: {last_status}")
+
+
+async def _post_onboarding_when_langflow_ready(
+ client: httpx.AsyncClient,
+ payload: dict[str, str],
+ timeout_s: float = 60.0,
+) -> httpx.Response:
+ deadline = asyncio.get_event_loop().time() + timeout_s
+ last_response = None
+ while asyncio.get_event_loop().time() < deadline:
+ response = await client.post("/onboarding", json=payload)
+ if response.status_code != 503 or "Langflow service" not in response.text:
+ return response
+ last_response = response
+ await asyncio.sleep(1.0)
+ assert last_response is not None
+ return last_response
+
+
+async def test_onboarding_ingests_sample_docs_and_creates_openrag_docs_filter(
+ isolated_onboarding_docs_workspace,
+):
+ from config.settings import clients, config_manager
+ from db.migrations_runtime import run_alembic_upgrade_async
+ from main import create_app
+
+ await run_alembic_upgrade_async("head")
+
+ app = await create_app()
+ startup_complete = False
+ try:
+ await app.router.startup()
+ startup_complete = True
+
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
+ response = await _post_onboarding_when_langflow_ready(
+ client,
+ {
+ "openai_api_key": os.environ["OPENAI_API_KEY"],
+ "llm_provider": "openai",
+ "embedding_provider": "openai",
+ "embedding_model": "text-embedding-3-small",
+ "llm_model": "gpt-4o-mini",
+ },
+ )
+
+ assert response.status_code == 200, response.text
+ payload = response.json()
+ assert payload["sample_data_ingested"] is True
+ assert payload["task_id"]
+ assert payload["openrag_docs_filter_id"]
+
+ task_status = await _wait_for_task(app.state.services["task_service"], payload["task_id"])
+ assert task_status["status"] == "completed"
+ assert task_status["successful_files"] == len(
+ isolated_onboarding_docs_workspace["expected_filenames"]
+ )
+ assert task_status["failed_files"] == 0
+
+ config = config_manager.get_config()
+ assert config.onboarding.openrag_docs_filter_id == payload["openrag_docs_filter_id"]
+ assert config.onboarding.openrag_docs_ingested_version
+
+ await clients.opensearch.indices.refresh(
+ index=isolated_onboarding_docs_workspace["index_name"]
+ )
+ search_response = await clients.opensearch.search(
+ index=isolated_onboarding_docs_workspace["index_name"],
+ body={
+ "query": {
+ "bool": {
+ "filter": [
+ {"term": {"connector_type": "openrag_docs"}},
+ {"term": {"is_sample_data": "true"}},
+ ]
+ }
+ },
+ "aggs": {"filenames": {"terms": {"field": "filename", "size": 100}}},
+ "size": 0,
+ },
+ )
+ total = search_response.get("hits", {}).get("total", {})
+ total_value = total.get("value", 0) if isinstance(total, dict) else total
+ assert total_value > 0, "Expected onboarding sample document chunks to be indexed"
+
+ filename_buckets = (
+ search_response.get("aggregations", {}).get("filenames", {}).get("buckets", [])
+ )
+ indexed_filenames = {bucket["key"] for bucket in filename_buckets}
+ assert set(isolated_onboarding_docs_workspace["expected_filenames"]).issubset(
+ indexed_filenames
+ )
+ assert indexed_filenames.isdisjoint(_EXCLUDED_DEFAULT_DOCS)
+ finally:
+ if startup_complete:
+ await app.router.shutdown()
diff --git a/tests/integration/core/test_runtime_migration.py b/tests/integration/core/test_runtime_migration.py
index feaa16049..9d0bc95b6 100644
--- a/tests/integration/core/test_runtime_migration.py
+++ b/tests/integration/core/test_runtime_migration.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import json
+import sys
from pathlib import Path
import httpx
@@ -12,6 +13,37 @@ import pytest_asyncio
import yaml
from sqlalchemy import func, select
+_RELOAD_MODULES = [
+ "api",
+ "api.connector_router",
+ "api.router",
+ "app.container",
+ "app.factory",
+ "app.lifespan",
+ "app.routes",
+ "app.routes.internal",
+ "auth_middleware",
+ "config.settings",
+ "dependencies",
+ "main",
+ "services",
+ "services.conversation_persistence_service",
+ "services.default_docs_service",
+ "services.rbac_service",
+ "services.search_service",
+ "services.session_ownership_service",
+ "services.startup_orchestrator",
+ "utils.opensearch_init",
+]
+
+_RELOAD_PREFIXES = ("api.", "app.", "services.")
+
+
+def _purge_reloaded_modules() -> None:
+ for mod in list(sys.modules):
+ if mod in _RELOAD_MODULES or mod.startswith(_RELOAD_PREFIXES):
+ sys.modules.pop(mod, None)
+
@pytest_asyncio.fixture
async def legacy_migration_workspace(tmp_path: Path, monkeypatch):
@@ -35,6 +67,11 @@ async def legacy_migration_workspace(tmp_path: Path, monkeypatch):
monkeypatch.setenv("DISABLE_STARTUP_INGEST", "true")
monkeypatch.setenv("FETCH_OPENRAG_DOCS_AT_STARTUP", "false")
+ from db.engine import dispose_engine as dispose_existing_engine
+
+ await dispose_existing_engine()
+ _purge_reloaded_modules()
+
from config.config_manager import config_manager
from db.engine import dispose_engine
from dependencies import invalidate_user_ensured_cache
@@ -80,6 +117,7 @@ async def legacy_migration_workspace(tmp_path: Path, monkeypatch):
session_ownership_service.ownership_file = old_ownership_file
session_ownership_service.ownership_data = old_ownership_data
session_ownership_service._session_factory = old_ownership_session_factory
+ _purge_reloaded_modules()
def _write_legacy_files(*, config_dir: Path, data_dir: Path) -> None:
@@ -210,12 +248,8 @@ async def _db_snapshot() -> dict[str, int]:
assert SessionLocal is not None
async with SessionLocal() as session:
users = await session.scalar(select(func.count()).select_from(User))
- conversations = await session.scalar(
- select(func.count()).select_from(Conversation)
- )
- ownership = await session.scalar(
- select(func.count()).select_from(SessionOwnership)
- )
+ conversations = await session.scalar(select(func.count()).select_from(Conversation))
+ ownership = await session.scalar(select(func.count()).select_from(SessionOwnership))
statuses = await session.scalar(select(func.count()).select_from(MigrationStatus))
return {
"users": int(users or 0),
@@ -287,9 +321,7 @@ async def test_legacy_file_state_migrates_on_backend_startup_and_is_idempotent()
first_snapshot = await _db_snapshot()
transport = httpx.ASGITransport(app=app)
- async with httpx.AsyncClient(
- transport=transport, base_url="http://testserver"
- ) as client:
+ async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
onboarding = await client.get("/onboarding-status")
assert onboarding.status_code == 200, onboarding.text
assert onboarding.json() == {"onboarded": True, "current_step": 4}
@@ -297,8 +329,7 @@ async def test_legacy_file_state_migrates_on_backend_startup_and_is_idempotent()
history = await client.get("/chat/history")
assert history.status_code == 200, history.text
conversations = {
- item["response_id"]: item
- for item in history.json().get("conversations", [])
+ item["response_id"]: item for item in history.json().get("conversations", [])
}
assert conversations["legacy-session"]["title"] == "Migrated chat"
assert conversations["legacy-session"]["total_messages"] == 3
@@ -310,9 +341,7 @@ async def test_legacy_file_state_migrates_on_backend_startup_and_is_idempotent()
try:
assert await _db_snapshot() == first_snapshot
transport = httpx.ASGITransport(app=app)
- async with httpx.AsyncClient(
- transport=transport, base_url="http://testserver"
- ) as client:
+ async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
onboarding = await client.get("/onboarding-status")
assert onboarding.status_code == 200, onboarding.text
assert onboarding.json()["onboarded"] is True
diff --git a/tests/integration/core/test_startup_ingest.py b/tests/integration/core/test_startup_ingest.py
index fca0c764f..230eafdb6 100644
--- a/tests/integration/core/test_startup_ingest.py
+++ b/tests/integration/core/test_startup_ingest.py
@@ -7,6 +7,35 @@ import pytest
# Files to exclude from ingestion (should match src/main.py)
EXCLUDED_INGESTION_FILES = {"warmup_ocr.pdf"}
+_RELOAD_MODULES = [
+ "api",
+ "api.router",
+ "api.connector_router",
+ "app",
+ "app.container",
+ "app.factory",
+ "app.lifespan",
+ "app.routes",
+ "app.routes.internal",
+ "config.settings",
+ "dependencies",
+ "auth_middleware",
+ "main",
+ "services",
+ "services.default_docs_service",
+ "services.search_service",
+ "services.startup_orchestrator",
+ "utils.opensearch_init",
+]
+_RELOAD_PREFIXES = ("api.", "app.", "services.")
+
+
+def _purge_reloaded_modules() -> None:
+ import sys
+
+ for mod in list(sys.modules):
+ if mod in _RELOAD_MODULES or mod.startswith(_RELOAD_PREFIXES):
+ sys.modules.pop(mod, None)
async def wait_for_ready(client: httpx.AsyncClient, timeout_s: float = 30.0):
@@ -32,7 +61,9 @@ def count_files_in_documents() -> int:
base_dir = Path(os.getcwd()) / "openrag-documents"
if not base_dir.is_dir():
return 0
- return sum(1 for _ in base_dir.rglob("*") if _.is_file() and _.name not in EXCLUDED_INGESTION_FILES)
+ return sum(
+ 1 for _ in base_dir.rglob("*") if _.is_file() and _.name not in EXCLUDED_INGESTION_FILES
+ )
@pytest.mark.parametrize("disable_langflow_ingest", [True, False])
@@ -40,27 +71,16 @@ def count_files_in_documents() -> int:
async def test_startup_ingest_creates_task(disable_langflow_ingest: bool):
# Ensure startup ingest runs and choose pipeline per param
os.environ["DISABLE_STARTUP_INGEST"] = "false"
- os.environ["DISABLE_INGEST_WITH_LANGFLOW"] = (
- "true" if disable_langflow_ingest else "false"
- )
+ os.environ["DISABLE_INGEST_WITH_LANGFLOW"] = "true" if disable_langflow_ingest else "false"
# Force no-auth mode for simpler endpoint access
os.environ["GOOGLE_OAUTH_CLIENT_ID"] = ""
os.environ["GOOGLE_OAUTH_CLIENT_SECRET"] = ""
# Reload settings to pick up env for this test run
- import sys
+ _purge_reloaded_modules()
- for mod in [
- "api.router",
- "api.connector_router",
- "config.settings",
- "auth_middleware",
- "main",
- ]:
- sys.modules.pop(mod, None)
-
- from main import create_app, startup_tasks
from config.settings import clients, get_index_name
+ from main import create_app
# Ensure a clean index before startup
await clients.initialize()
@@ -70,15 +90,17 @@ async def test_startup_ingest_creates_task(disable_langflow_ingest: bool):
pass
app = await create_app()
- # Trigger startup tasks explicitly
- await startup_tasks(app.state.services)
+ startup_complete = False
+ try:
+ await app.router.startup()
+ startup_complete = True
- # Ensure index exists for tests (startup_tasks only creates it if DISABLE_INGEST_WITH_LANGFLOW=True)
- from main import _ensure_opensearch_index
- await _ensure_opensearch_index()
+ # Ensure index exists for tests (startup_tasks only creates it if DISABLE_INGEST_WITH_LANGFLOW=True)
+ from main import _ensure_opensearch_index
- transport = httpx.ASGITransport(app=app)
- try:
+ await _ensure_opensearch_index()
+
+ transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
await wait_for_ready(client)
@@ -107,14 +129,24 @@ async def test_startup_ingest_creates_task(disable_langflow_ingest: bool):
sr = await client.post("/search", json={"query": "*", "limit": 1})
assert sr.status_code == 200, sr.text
total = sr.json().get("total")
- assert isinstance(total, int) and total >= 0, "Startup ingest did not index documents"
+ assert isinstance(total, int) and total >= 0, (
+ "Startup ingest did not index documents"
+ )
return
newest = tasks[0]
assert "task_id" in newest
- assert newest.get("total_files") == expected_files
+ assert isinstance(newest.get("total_files"), int)
+ assert newest["total_files"] > 0
+ assert newest.get("files")
finally:
+ if startup_complete:
+ try:
+ await app.router.shutdown()
+ except Exception:
+ pass
# Explicitly close global clients to avoid aiohttp warnings
from config.settings import clients
+
try:
await clients.close()
except Exception:
diff --git a/tests/integration/sdk/conftest.py b/tests/integration/sdk/conftest.py
index 5e48fb95c..711e75d58 100644
--- a/tests/integration/sdk/conftest.py
+++ b/tests/integration/sdk/conftest.py
@@ -6,6 +6,7 @@ Set OPENRAG_URL (default: http://localhost:3000) before running.
"""
import os
+import time
import uuid
from pathlib import Path
@@ -78,12 +79,36 @@ async def client():
from openrag_sdk import OpenRAGClient
api_key = await _fetch_api_key()
+
+ async def log_request(request: httpx.Request) -> None:
+ request_id = f"sdk-{uuid.uuid4().hex}"
+ request.headers["x-request-id"] = request_id
+ request.extensions["openrag_request_id"] = request_id
+ request.extensions["openrag_started_at"] = time.perf_counter()
+ print(f"[SDK HTTP] start request_id={request_id} method={request.method} url={request.url}")
+
+ async def log_response(response: httpx.Response) -> None:
+ started_at = response.request.extensions.get("openrag_started_at")
+ duration_ms = (
+ round((time.perf_counter() - started_at) * 1000)
+ if isinstance(started_at, float)
+ else None
+ )
+ request_id = response.request.extensions.get("openrag_request_id")
+ print(
+ "[SDK HTTP] response "
+ f"request_id={request_id} status={response.status_code} duration_ms={duration_ms}"
+ )
+
# The SDK defaults to a 30s timeout for *all* requests. Streaming chat on a
# cold CI box (model spin-up + flow init before the first byte) routinely
# exceeds that, surfacing as httpx.ReadTimeout. Use a generous timeout here.
- c = OpenRAGClient(api_key=api_key, base_url=_base_url, timeout=120.0)
- yield c
- await c.close()
+ async with httpx.AsyncClient(
+ timeout=120.0,
+ event_hooks={"request": [log_request], "response": [log_response]},
+ ) as http_client:
+ c = OpenRAGClient(api_key=api_key, base_url=_base_url, http_client=http_client)
+ yield c
@pytest.fixture
diff --git a/tests/integration/sdk/test_chat.py b/tests/integration/sdk/test_chat.py
index 501337c87..8955f8a4b 100644
--- a/tests/integration/sdk/test_chat.py
+++ b/tests/integration/sdk/test_chat.py
@@ -22,6 +22,8 @@ class TestChat:
assert response.response is not None
assert isinstance(response.response, str)
assert len(response.response) > 0
+ assert response.sources is not None
+ assert isinstance(response.sources, list)
@pytest.mark.asyncio
async def test_chat_streaming_create(self, client):
@@ -110,6 +112,7 @@ class TestChat:
assert result is True
@pytest.mark.asyncio
+ @pytest.mark.skip(reason="Structured source extraction is being fixed in a follow-up PR")
async def test_chat_with_sources(self, client, test_file: Path):
"""Chat response must cite the ingested document as a source (RAG)."""
result = await client.documents.ingest(file_path=str(test_file))
@@ -162,20 +165,8 @@ class TestChatExtended:
await stream.final_text()
assert stream.chat_id is not None
- @pytest.mark.asyncio
- async def test_chat_sources_field_is_list(self, client):
- """sources on ChatResponse is always a list (may be empty)."""
- response = await client.chat.create(message="What time is it?")
- assert response.sources is not None
- assert isinstance(response.sources, list)
- if response.chat_id:
- await client.chat.delete(response.chat_id)
-
@pytest.mark.asyncio
async def test_list_conversations_returns_list(self, client):
"""list() always returns a ConversationListResponse with a list."""
- r = await client.chat.create(message="List test message.")
result = await client.chat.list()
assert isinstance(result.conversations, list)
- if r.chat_id:
- await client.chat.delete(r.chat_id)
diff --git a/tests/unit/api/test_acl.py b/tests/unit/api/test_acl.py
index 72383c0a2..78c659aba 100644
--- a/tests/unit/api/test_acl.py
+++ b/tests/unit/api/test_acl.py
@@ -38,6 +38,14 @@ async def test_get_document_acl_returns_read_only_acl_for_document():
"owner": "owner",
"allowed_users": ["reader"],
"allowed_groups": ["engineering"],
+ "allowed_principal_labels": [
+ {
+ "principal": "g:gdrive:tenant:engineering",
+ "kind": "group",
+ "provider": "gdrive",
+ "display_name": "Engineering",
+ }
+ ],
}
}
]
@@ -56,9 +64,22 @@ async def test_get_document_acl_returns_read_only_acl_for_document():
"owner": "owner",
"allowed_users": ["reader"],
"allowed_groups": ["engineering"],
+ "allowed_principal_labels": [
+ {
+ "principal": "g:gdrive:tenant:engineering",
+ "kind": "group",
+ "provider": "gdrive",
+ "display_name": "Engineering",
+ }
+ ],
}
search_body = opensearch_client.search.await_args.kwargs["body"]
- assert search_body["_source"] == ["owner", "allowed_users", "allowed_groups"]
+ assert search_body["_source"] == [
+ "owner",
+ "allowed_users",
+ "allowed_groups",
+ "allowed_principal_labels",
+ ]
def test_document_share_methods_are_not_exposed():
diff --git a/tests/unit/api/test_delete_chunks_by_document_ids.py b/tests/unit/api/test_delete_chunks_by_document_ids.py
index 210e79612..a292277ca 100644
--- a/tests/unit/api/test_delete_chunks_by_document_ids.py
+++ b/tests/unit/api/test_delete_chunks_by_document_ids.py
@@ -40,22 +40,27 @@ async def test_deletes_each_visible_chunk_id_by_primary_id():
from api.documents import delete_chunks_by_document_ids
opensearch_client = AsyncMock()
+ write_client = AsyncMock()
# Visible chunk _ids for the requested document_ids.
chunk_ids = ["chunk-1", "chunk-2", "chunk-3"]
opensearch_client.search.return_value = {
"_scroll_id": None,
"hits": {"hits": [{"_id": cid} for cid in chunk_ids]},
}
- opensearch_client.delete.return_value = {"result": "deleted"}
+ write_client.delete.return_value = {"result": "deleted"}
deleted = await delete_chunks_by_document_ids(
- ["doc-a", "doc-b"], opensearch_client, "test-index"
+ ["doc-a", "doc-b"],
+ opensearch_client,
+ "test-index",
+ write_opensearch_client=write_client,
)
assert deleted == len(chunk_ids)
# delete_by_query must NOT be used — silently filtered under DLS.
opensearch_client.delete_by_query.assert_not_awaited()
+ write_client.delete_by_query.assert_not_awaited()
# Search query must target the document_id field with terms(...).
search_call = opensearch_client.search.await_args
@@ -64,7 +69,8 @@ async def test_deletes_each_visible_chunk_id_by_primary_id():
# One primary-id delete per visible chunk, refresh=True so the delete is
# immediately visible to the re-index that typically follows.
- delete_calls = opensearch_client.delete.await_args_list
+ opensearch_client.delete.assert_not_awaited()
+ delete_calls = write_client.delete.await_args_list
assert len(delete_calls) == len(chunk_ids)
for call, cid in zip(delete_calls, chunk_ids, strict=True):
assert call.kwargs["index"] == "test-index"
@@ -79,36 +85,45 @@ async def test_returns_zero_when_no_visible_chunks_match():
from api.documents import delete_chunks_by_document_ids
opensearch_client = AsyncMock()
+ write_client = AsyncMock()
opensearch_client.search.return_value = {"_scroll_id": None, "hits": {"hits": []}}
- deleted = await delete_chunks_by_document_ids(["abc"], opensearch_client, "test-index")
+ deleted = await delete_chunks_by_document_ids(
+ ["abc"],
+ opensearch_client,
+ "test-index",
+ write_opensearch_client=write_client,
+ )
assert deleted == 0
opensearch_client.delete.assert_not_awaited()
+ write_client.delete.assert_not_awaited()
@pytest.mark.asyncio
async def test_custom_field_parameter_is_used_in_query():
- """Passing field='connector_file_id' must produce a terms query against that
- field instead of the default 'document_id'. This is required for the 404
- cleanup path in ConnectorFileProcessor where chunks are indexed with
- document_id=file_hash (SHA) but deleted by connector source file ID."""
+ """Passing field='connector_file_id' must query that field and still delete
+ through the trusted backend write client."""
from api.documents import delete_chunks_by_document_ids
opensearch_client = AsyncMock()
+ write_client = AsyncMock()
opensearch_client.search.return_value = {
"_scroll_id": None,
"hits": {"hits": [{"_id": "chunk-x"}]},
}
- opensearch_client.delete.return_value = {"result": "deleted"}
+ write_client.delete.return_value = {"result": "deleted"}
deleted = await delete_chunks_by_document_ids(
["src-file-1"],
opensearch_client,
"test-index",
+ write_opensearch_client=write_client,
field="connector_file_id",
)
assert deleted == 1
search_call = opensearch_client.search.await_args
assert search_call.kwargs["body"]["query"] == {"terms": {"connector_file_id": ["src-file-1"]}}
+ opensearch_client.delete.assert_not_awaited()
+ write_client.delete.assert_awaited_once()
diff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
index c7f2a186f..e7ff1dca8 100644
--- a/tests/unit/api/test_reconcile_orphans_for_connector_type.py
+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
@@ -1,17 +1,12 @@
"""Unit tests for `reconcile_orphans_for_connector_type` in `src/api/connectors.py`.
-This is the orphan-deletion safety net for the connector sync flow. The
-function must:
-- Compute orphans = indexed_ids - union(remote_ids across all active
- connections of this connector_type for this user).
-- Apply STRICT gating: any unauthenticated connection or listing
- exception aborts the pass with 0 deletes (false-negative > false-positive).
-- Preserve files that exist in any active connection of the type
- (multi-connection isolation).
-- Delete orphan chunks using the DLS-safe enumerate-then-delete pattern
- (NOT delete_by_query, which is silently no-opped under DLS).
-- Query the correct OpenSearch field: connector_file_id for non-Langflow
- chunks, document_id for Langflow chunks.
+The orphan-deletion safety net must:
+- compute orphans from indexed IDs minus the union of active remote IDs,
+- abort on unauthenticated or failing connector listings,
+- preserve files present in any active connection,
+- enumerate visible chunks with the user-scoped client, then delete by primary
+ ID with the trusted backend OpenSearch client,
+- query either `document_id` or `connector_file_id` depending on the ingest path.
"""
import sys
@@ -56,16 +51,11 @@ def _make_service(connections, connector_lookup):
def _make_opensearch_client(chunk_ids: list[str] | None = None):
- """Build an OpenSearch mock wired for the enumerate-then-delete pattern.
-
- ``chunk_ids`` controls which chunk _ids the search returns (simulating
- what collect_visible_document_ids finds). Pass ``None`` or ``[]`` for
- "no chunks found" (no-orphan scenarios).
- """
client = AsyncMock()
hits = [{"_id": cid} for cid in (chunk_ids or [])]
client.search = AsyncMock(return_value={"_scroll_id": None, "hits": {"hits": hits}})
client.delete = AsyncMock(return_value={"result": "deleted"})
+ client.delete_by_query = AsyncMock()
return client
@@ -75,9 +65,16 @@ def _make_session_manager(opensearch_client):
return sm
-# ---------------------------------------------------------------------------
-# Gating / no-op scenarios
-# ---------------------------------------------------------------------------
+def _patch_write_client(monkeypatch, *, delete_side_effect=None):
+ write_client = AsyncMock()
+ write_client.delete = AsyncMock(
+ side_effect=delete_side_effect,
+ return_value={"result": "deleted"},
+ )
+ write_client.delete_by_query = AsyncMock()
+ monkeypatch.setattr("config.settings.clients.opensearch", write_client)
+ monkeypatch.setattr("api.connectors.get_index_name", lambda: "test-index")
+ return write_client
@pytest.mark.asyncio
@@ -126,7 +123,6 @@ async def test_no_active_connections_skips_reconcile():
@pytest.mark.asyncio
async def test_unauthenticated_connection_aborts_pass():
- """STRICT GATING: even one unauthenticated connector aborts the pass."""
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
@@ -151,7 +147,6 @@ async def test_unauthenticated_connection_aborts_pass():
@pytest.mark.asyncio
async def test_listing_exception_aborts_pass():
- """STRICT GATING: a transient list_files error aborts the pass."""
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
@@ -175,7 +170,6 @@ async def test_listing_exception_aborts_pass():
@pytest.mark.asyncio
async def test_no_orphans_skips_delete_call():
- """If every indexed ID still exists remotely, no delete must be issued."""
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
@@ -197,26 +191,15 @@ async def test_no_orphans_skips_delete_call():
opensearch_client.delete.assert_not_awaited()
-# ---------------------------------------------------------------------------
-# Happy path (DLS-safe enumerate-then-delete)
-# ---------------------------------------------------------------------------
-
-
@pytest.mark.asyncio
-async def test_happy_path_deletes_orphans():
- """Indexed has [a, b, c]; remote has [a, c] → orphan = [b].
-
- The implementation must use the DLS-safe pattern: enumerate visible chunk
- _ids via search, then issue one primary-id delete per chunk. delete_by_query
- must NEVER be called (silently no-opped under DLS).
- """
+async def test_happy_path_deletes_orphans(monkeypatch):
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
connector = _make_connector(remote_file_ids=["a", "c"])
service = _make_service([conn], connector_lookup={"c1": connector})
- # One chunk belongs to orphan "b"
- opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-0"])
+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-1", "chunk-b-2"])
+ write_client = _patch_write_client(monkeypatch)
sm = _make_session_manager(opensearch_client)
result = await reconcile_orphans_for_connector_type(
@@ -229,40 +212,29 @@ async def test_happy_path_deletes_orphans():
)
assert result == ["b"]
+ opensearch_client.delete_by_query.assert_not_awaited()
+ write_client.delete_by_query.assert_not_awaited()
+ opensearch_client.delete.assert_not_awaited()
- # DLS-safe: must NOT call delete_by_query
- opensearch_client.delete_by_query.assert_not_called()
-
- # Must enumerate via search first…
- opensearch_client.search.assert_awaited_once()
search_body = opensearch_client.search.await_args.kwargs["body"]
assert search_body["query"] == {"terms": {"document_id": ["b"]}}
-
- # …then delete each chunk by primary _id
- opensearch_client.delete.assert_awaited_once()
- delete_kwargs = opensearch_client.delete.await_args.kwargs
- assert delete_kwargs["id"] == "chunk-b-0"
- assert delete_kwargs.get("refresh") is True
+ assert [call.kwargs["id"] for call in write_client.delete.await_args_list] == [
+ "chunk-b-1",
+ "chunk-b-2",
+ ]
@pytest.mark.asyncio
-async def test_delete_failure_does_not_raise():
- """If the bulk delete blows up, the helper must swallow it (no exception).
- The function still returns the orphan IDs that were identified — callers
- use the return value to know *what* was orphaned, not whether deletion
- succeeded."""
+async def test_delete_failure_does_not_raise(monkeypatch):
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
connector = _make_connector(remote_file_ids=["a"])
service = _make_service([conn], connector_lookup={"c1": connector})
-
- opensearch_client = AsyncMock()
- # Make the search call itself fail so delete_chunks_by_document_ids raises
- opensearch_client.search = AsyncMock(side_effect=RuntimeError("opensearch unavailable"))
+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-1"])
+ _patch_write_client(monkeypatch, delete_side_effect=RuntimeError("opensearch unavailable"))
sm = _make_session_manager(opensearch_client)
- # Must not raise even though deletion fails
result = await reconcile_orphans_for_connector_type(
connector_type="sharepoint",
user_id="alice",
@@ -272,18 +244,11 @@ async def test_delete_failure_does_not_raise():
existing_file_ids=["a", "b"],
)
- # "b" was identified as orphaned; deletion failed silently
- assert result == ["b"]
-
-
-# ---------------------------------------------------------------------------
-# Multi-connection isolation
-# ---------------------------------------------------------------------------
+ assert result == []
@pytest.mark.asyncio
async def test_multi_connection_union_preserves_files_present_in_any_connection():
- """conn-A has [a]. conn-B has [b]. Both [a, b] must be preserved."""
from api.connectors import reconcile_orphans_for_connector_type
conn_a = _make_connection("conn-a")
@@ -312,7 +277,6 @@ async def test_multi_connection_union_preserves_files_present_in_any_connection(
@pytest.mark.asyncio
async def test_multi_connection_one_offline_aborts_even_if_other_succeeds():
- """conn-B unauthenticated — must abort even though conn-A succeeded."""
from api.connectors import reconcile_orphans_for_connector_type
conn_a = _make_connection("conn-a")
@@ -341,7 +305,6 @@ async def test_multi_connection_one_offline_aborts_even_if_other_succeeds():
@pytest.mark.asyncio
async def test_paginated_listing_aggregates_all_pages():
- """Remote listings are paginated — must walk every page."""
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
@@ -371,24 +334,15 @@ async def test_paginated_listing_aggregates_all_pages():
opensearch_client.delete.assert_not_awaited()
-# ---------------------------------------------------------------------------
-# id_field routing — connector_file_id vs document_id
-# ---------------------------------------------------------------------------
-
-
@pytest.mark.asyncio
-async def test_connector_file_id_field_used_when_specified():
- """When id_field='connector_file_id', the delete query must target that
- field. This is the non-Langflow (DISABLE_INGEST_WITH_LANGFLOW=True) path
- where chunks carry document_id=SHA_hash but connector_file_id=connector_ID.
- """
+async def test_connector_file_id_field_used_when_specified(monkeypatch):
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
- # Remote only has "sp-guid-a"; "sp-guid-b" is orphaned
connector = _make_connector(remote_file_ids=["sp-guid-a"])
service = _make_service([conn], connector_lookup={"c1": connector})
opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-0", "chunk-b-1"])
+ write_client = _patch_write_client(monkeypatch)
sm = _make_session_manager(opensearch_client)
result = await reconcile_orphans_for_connector_type(
@@ -402,28 +356,21 @@ async def test_connector_file_id_field_used_when_specified():
)
assert result == ["sp-guid-b"]
-
- opensearch_client.delete_by_query.assert_not_called()
-
search_body = opensearch_client.search.await_args.kwargs["body"]
- assert search_body["query"] == {"terms": {"connector_file_id": ["sp-guid-b"]}}, (
- f"Expected connector_file_id query, got: {search_body['query']}"
- )
-
- assert opensearch_client.delete.await_count == 2
+ assert search_body["query"] == {"terms": {"connector_file_id": ["sp-guid-b"]}}
+ assert write_client.delete.await_count == 2
+ opensearch_client.delete.assert_not_awaited()
@pytest.mark.asyncio
-async def test_document_id_field_used_by_default():
- """Default id_field='document_id' preserves the Langflow path behavior
- where document_id already holds the connector source ID.
- """
+async def test_document_id_field_used_by_default(monkeypatch):
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
connector = _make_connector(remote_file_ids=["lf-id-a"])
service = _make_service([conn], connector_lookup={"c1": connector})
opensearch_client = _make_opensearch_client(chunk_ids=["chunk-lf-b-0"])
+ write_client = _patch_write_client(monkeypatch)
sm = _make_session_manager(opensearch_client)
result = await reconcile_orphans_for_connector_type(
@@ -433,12 +380,9 @@ async def test_document_id_field_used_by_default():
session_manager=sm,
jwt_token=None,
existing_file_ids=["lf-id-a", "lf-id-b"],
- # id_field defaults to "document_id"
)
assert result == ["lf-id-b"]
-
search_body = opensearch_client.search.await_args.kwargs["body"]
- assert search_body["query"] == {"terms": {"document_id": ["lf-id-b"]}}, (
- f"Expected document_id query, got: {search_body['query']}"
- )
+ assert search_body["query"] == {"terms": {"document_id": ["lf-id-b"]}}
+ write_client.delete.assert_awaited_once()
diff --git a/tests/unit/connectors/test_update_connector_metadata_filename.py b/tests/unit/connectors/test_update_connector_metadata_filename.py
index a98f375f9..cd553dbd1 100644
--- a/tests/unit/connectors/test_update_connector_metadata_filename.py
+++ b/tests/unit/connectors/test_update_connector_metadata_filename.py
@@ -31,6 +31,10 @@ def _make_service():
opensearch_client = AsyncMock()
service.session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+ service.clients = MagicMock()
+ # Keep the same mock for visibility checks and trusted writes so these
+ # tests can focus on the update_by_query body.
+ service.clients.opensearch = opensearch_client
return service, opensearch_client
diff --git a/tests/unit/dependencies/test_auth_jwt_attachment.py b/tests/unit/dependencies/test_auth_jwt_attachment.py
new file mode 100644
index 000000000..8f5b8102e
--- /dev/null
+++ b/tests/unit/dependencies/test_auth_jwt_attachment.py
@@ -0,0 +1,94 @@
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+import dependencies # noqa: E402
+from dependencies import get_current_user, get_optional_user # noqa: E402
+from session_manager import User # noqa: E402
+
+
+class FakeSessionManager:
+ def __init__(self):
+ self.users: dict[str, User] = {}
+ self.effective_token_calls: list[tuple[str, str | None]] = []
+
+ def get_effective_jwt_token(self, user_id: str, jwt_token: str | None) -> str:
+ self.effective_token_calls.append((user_id, jwt_token))
+ return jwt_token or "Bearer generated-anonymous-token"
+
+ def get_user_from_token(self, token: str) -> User | None:
+ return User(
+ user_id="user-1",
+ name="Test User",
+ provider="google",
+ )
+
+
+def _request(cookies: dict[str, str] | None = None):
+ return SimpleNamespace(
+ cookies=cookies or {},
+ headers={},
+ state=SimpleNamespace(),
+ app=SimpleNamespace(state=SimpleNamespace(services={})),
+ )
+
+
[email protected](autouse=True)
+def _auth_settings(monkeypatch):
+ monkeypatch.setattr("config.settings.IBM_AUTH_ENABLED", False)
+
+ async def _resolve_db_user_id(user):
+ return user.user_id
+
+ monkeypatch.setattr(dependencies, "_resolve_db_user_id", _resolve_db_user_id)
+
+
+async def test_current_user_no_auth_gets_effective_jwt(monkeypatch):
+ monkeypatch.setattr("config.settings.is_no_auth_mode", lambda: True)
+ session_manager = FakeSessionManager()
+ request = _request()
+
+ user = await get_current_user(request, session_manager=session_manager)
+
+ assert user.user_id == "anonymous"
+ assert user.jwt_token == "Bearer generated-anonymous-token"
+ assert request.state.user.jwt_token == "Bearer generated-anonymous-token"
+ assert session_manager.effective_token_calls == [("anonymous", None)]
+
+
+async def test_optional_user_no_auth_gets_effective_jwt(monkeypatch):
+ monkeypatch.setattr("config.settings.is_no_auth_mode", lambda: True)
+ session_manager = FakeSessionManager()
+ request = _request()
+
+ user = await get_optional_user(request, session_manager=session_manager)
+
+ assert user is not None
+ assert user.user_id == "anonymous"
+ assert user.jwt_token == "Bearer generated-anonymous-token"
+ assert request.state.user.jwt_token == "Bearer generated-anonymous-token"
+ assert session_manager.effective_token_calls == [("anonymous", None)]
+
+
+async def test_current_user_cookie_token_is_attached_for_opensearch(monkeypatch):
+ monkeypatch.setattr("config.settings.is_no_auth_mode", lambda: False)
+ session_manager = FakeSessionManager()
+ request = _request(cookies={"auth_token": "Bearer session-token"})
+
+ user = await get_current_user(request, session_manager=session_manager)
+
+ assert user.user_id == "user-1"
+ assert user.jwt_token == "Bearer session-token"
+ assert request.state.user.jwt_token == "Bearer session-token"
+ assert session_manager.effective_token_calls == [("user-1", "Bearer session-token")]
diff --git a/tests/unit/dependencies/test_ibm_auth.py b/tests/unit/dependencies/test_ibm_auth.py
new file mode 100644
index 000000000..9552951f9
--- /dev/null
+++ b/tests/unit/dependencies/test_ibm_auth.py
@@ -0,0 +1,60 @@
+import base64
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+from dependencies import _get_ibm_user # noqa: E402
+
+
+async def test_ibm_header_auth_uses_opensearch_username_without_session_cookie(monkeypatch):
+ monkeypatch.setattr("config.settings.IBM_CREDENTIALS_HEADER", "X-Test-IBM-Credentials")
+ monkeypatch.setattr("config.settings.IBM_SESSION_COOKIE_NAME", "ibm-openrag-session")
+ monkeypatch.setattr("config.settings.PLATFORM_USERNAME", "")
+ monkeypatch.setattr("config.settings.PLATFORM_PASSWORD", "")
+
+ class ConnectionManager:
+ def __init__(self):
+ self.upserts = []
+
+ async def upsert_ibm_credentials(self, **kwargs):
+ self.upserts.append(kwargs)
+
+ connection_manager = ConnectionManager()
+ request = SimpleNamespace(
+ headers={
+ "X-Test-IBM-Credentials": "Basic "
+ + base64.b64encode(b"ibmlhapikey_user-1:secret").decode()
+ },
+ cookies={},
+ state=SimpleNamespace(),
+ app=SimpleNamespace(
+ state=SimpleNamespace(
+ services={
+ "connector_service": SimpleNamespace(connection_manager=connection_manager)
+ }
+ )
+ ),
+ )
+
+ user = await _get_ibm_user(request, required=True)
+
+ assert user.user_id == "ibmlhapikey_user-1"
+ assert user.email == "ibmlhapikey_user-1"
+ assert user.name == "ibmlhapikey_user-1"
+ assert user.opensearch_username == "ibmlhapikey_user-1"
+ assert request.state.user == user
+ assert connection_manager.upserts == [
+ {
+ "user_id": "ibmlhapikey_user-1",
+ "basic_credentials": request.headers["X-Test-IBM-Credentials"],
+ "username": "ibmlhapikey_user-1",
+ }
+ ]
diff --git a/tests/unit/services/test_default_docs_jwt.py b/tests/unit/services/test_default_docs_jwt.py
new file mode 100644
index 000000000..d8d6151bd
--- /dev/null
+++ b/tests/unit/services/test_default_docs_jwt.py
@@ -0,0 +1,72 @@
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+from services.default_docs_service import ( # noqa: E402
+ _ingest_default_documents_langflow,
+ _ingest_default_documents_url_langflow,
+)
+
+
+class FakeSessionManager:
+ def __init__(self):
+ self.calls: list[tuple[str, str | None]] = []
+
+ def get_effective_jwt_token(self, user_id: str, jwt_token: str | None) -> str:
+ self.calls.append((user_id, jwt_token))
+ return "Bearer default-doc-token"
+
+
+class FakeTaskService:
+ def __init__(self):
+ self.upload_kwargs = None
+ self.url_kwargs = None
+
+ async def create_langflow_upload_task(self, **kwargs):
+ self.upload_kwargs = kwargs
+ return "upload-task"
+
+ async def create_langflow_url_upload_task(self, **kwargs):
+ self.url_kwargs = kwargs
+ return "url-task"
+
+
+async def test_default_file_docs_use_effective_jwt_helper():
+ session_manager = FakeSessionManager()
+ task_service = FakeTaskService()
+
+ task_id = await _ingest_default_documents_langflow(
+ langflow_file_service=object(),
+ session_manager=session_manager,
+ task_service=task_service,
+ file_paths=["/tmp/openrag-doc.md"],
+ )
+
+ assert task_id == "upload-task"
+ assert session_manager.calls == [("anonymous", None)]
+ assert task_service.upload_kwargs["jwt_token"] == "Bearer default-doc-token"
+
+
+async def test_default_url_docs_use_effective_jwt_helper():
+ session_manager = FakeSessionManager()
+ task_service = FakeTaskService()
+
+ task_id = await _ingest_default_documents_url_langflow(
+ langflow_file_service=object(),
+ session_manager=session_manager,
+ task_service=task_service,
+ docs_url="https://docs.example.test",
+ crawl_depth=1,
+ )
+
+ assert task_id == "url-task"
+ assert session_manager.calls == [("anonymous", None)]
+ assert task_service.url_kwargs["jwt_token"] == "Bearer default-doc-token"
diff --git a/tests/unit/services/test_langflow_mcp_service.py b/tests/unit/services/test_langflow_mcp_service.py
new file mode 100644
index 000000000..f2db7d125
--- /dev/null
+++ b/tests/unit/services/test_langflow_mcp_service.py
@@ -0,0 +1,118 @@
+import pytest
+
+import services.langflow_mcp_service as mcp_module
+from services.langflow_mcp_service import (
+ LangflowMCPService,
+ MCPServerURLUpdateError,
+)
+
+
+class _Response:
+ def __init__(self, status_code: int, text: str = "") -> None:
+ self.status_code = status_code
+ self.text = text
+
+
+async def test_update_all_mcp_server_urls_strict_raises_with_failed_server(
+ monkeypatch,
+):
+ service = LangflowMCPService()
+
+ async def fake_list_mcp_servers(*, raise_on_error: bool = False):
+ assert raise_on_error is True
+ return [{"name": "lf-starter_project"}]
+
+ async def fake_patch_mcp_server_url(server_name: str):
+ assert server_name == "lf-starter_project"
+ return "failed"
+
+ monkeypatch.setattr(service, "list_mcp_servers", fake_list_mcp_servers)
+ monkeypatch.setattr(service, "patch_mcp_server_url", fake_patch_mcp_server_url)
+
+ with pytest.raises(MCPServerURLUpdateError) as exc_info:
+ await service.update_all_mcp_server_urls(strict=True)
+
+ assert exc_info.value.summary["failed"] == 1
+ assert exc_info.value.summary["failed_servers"] == ["lf-starter_project"]
+ assert "lf-starter_project" in str(exc_info.value)
+
+
+async def test_patch_mcp_server_url_retries_retryable_patch_status(monkeypatch):
+ service = LangflowMCPService()
+ service.patch_retry_base_seconds = 0
+ service.patch_retry_max_seconds = 0
+ monkeypatch.setenv("LANGFLOW_URL", "http://langflow:7860")
+
+ async def fake_get_mcp_server(server_name: str):
+ assert server_name == "lf-starter_project"
+ return {"url": "http://localhost:8000/mcp"}
+
+ patch_requests = []
+
+ async def fake_langflow_request(**kwargs):
+ patch_requests.append(kwargs)
+ if len(patch_requests) < 3:
+ return _Response(503, "Langflow warming up")
+ return _Response(200, "ok")
+
+ monkeypatch.setattr(service, "get_mcp_server", fake_get_mcp_server)
+ monkeypatch.setattr(
+ mcp_module.clients,
+ "langflow_request",
+ fake_langflow_request,
+ raising=True,
+ )
+
+ result = await service.patch_mcp_server_url("lf-starter_project")
+
+ assert result == "patched"
+ assert len(patch_requests) == 3
+ assert patch_requests[-1]["json"] == {"url": "http://langflow:7860/mcp"}
+
+
+async def test_patch_mcp_server_url_preserves_auth_settings(monkeypatch):
+ service = LangflowMCPService()
+ monkeypatch.setenv("LANGFLOW_URL", "http://langflow:7860")
+
+ async def fake_get_mcp_server(server_name: str):
+ assert server_name == "lf-starter_project"
+ return {
+ "url": "http://localhost:7860/api/v1/mcp/project/project-id/streamable",
+ "auth_type": "api_key",
+ "headers": {
+ "x-api-key": "langflow-key",
+ "X-Langflow-Global-Var-JWT": "JWT",
+ },
+ }
+
+ patch_requests = []
+
+ async def fake_langflow_request(**kwargs):
+ patch_requests.append(kwargs)
+ return _Response(200, "ok")
+
+ monkeypatch.setattr(service, "get_mcp_server", fake_get_mcp_server)
+ monkeypatch.setattr(
+ mcp_module.clients,
+ "langflow_request",
+ fake_langflow_request,
+ raising=True,
+ )
+
+ result = await service.patch_mcp_server_url("lf-starter_project")
+
+ assert result == "patched"
+ assert patch_requests == [
+ {
+ "method": "PATCH",
+ "endpoint": "/api/v2/mcp/servers/lf-starter_project",
+ "json": {
+ "url": "http://langflow:7860/api/v1/mcp/project/project-id/streamable",
+ "auth_type": "api_key",
+ "headers": {"x-api-key": "langflow-key"},
+ },
+ }
+ ]
diff --git a/tests/unit/services/test_user_service_bootstrap.py b/tests/unit/services/test_user_service_bootstrap.py
index 079647f77..4745978c3 100644
--- a/tests/unit/services/test_user_service_bootstrap.py
+++ b/tests/unit/services/test_user_service_bootstrap.py
@@ -89,6 +89,38 @@ async def test_repeated_calls_are_idempotent(session, monkeypatch):
assert {r.name for r in roles} == {"user"}
+async def test_existing_user_without_roles_is_repaired_on_signin(session, monkeypatch):
+ monkeypatch.setenv("OPENRAG_DEFAULT_ROLE", "user")
+ await ensure_user_row(session, _user(uid="oauth-admin", email="[email protected]"))
+ await session.commit()
+
+ from db.models import User as UserRow
+ from db.repositories._helpers import email_lookup_hash
+
+ row = UserRow(
+ id="roleless-user",
+ oauth_provider="google",
+ oauth_subject="roleless-user",
+ email_lookup_hash=email_lookup_hash("[email protected]"),
+ display_name="Roleless",
+ )
+ session.add(row)
+ await session.commit()
+
+ repaired = await ensure_user_row(
+ session,
+ _user(uid="roleless-user", email="[email protected]", name="Roleless"),
+ )
+ await session.commit()
+
+ assert repaired.id == "roleless-user"
+ role_repo = RoleRepo(session)
+ roles = {r.name for r in await role_repo.list_user_roles(repaired.id)}
+ assert roles == {"user"}
+
+
@pytest.mark.asyncio
async def test_new_user_id_matches_oauth_subject(session):
"""Regression: the SQL users.id must equal the OAuth subject so
diff --git a/tests/unit/test_agent_response.py b/tests/unit/test_agent_response.py
new file mode 100644
index 000000000..5e8a0aa16
--- /dev/null
+++ b/tests/unit/test_agent_response.py
@@ -0,0 +1,84 @@
+from types import SimpleNamespace
+
+import pytest
+
+from agent import _extract_delta_text, async_response
+
+
+class BrokenOutputTextResponse:
+ id = "non-stream-response"
+ error = None
+
+ @property
+ def output_text(self):
+ raise TypeError("'NoneType' object is not iterable")
+
+
+class Chunk(SimpleNamespace):
+ def model_dump(self):
+ return self.__dict__
+
+
+class AsyncChunkStream:
+ def __init__(self, chunks):
+ self._chunks = iter(chunks)
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ try:
+ return next(self._chunks)
+ except StopIteration:
+ raise StopAsyncIteration from None
+
+
+class FakeResponses:
+ def __init__(self):
+ self.calls = []
+
+ async def create(self, **kwargs):
+ self.calls.append(kwargs)
+ if not kwargs["stream"]:
+ return BrokenOutputTextResponse()
+
+ return AsyncChunkStream(
+ [
+ Chunk(type="response.output_text.delta", delta={"content": "Hello"}),
+ Chunk(type="response.output_text.delta", delta={"content": " world"}),
+ Chunk(
+ type="response.completed",
+ response={"id": "stream-response", "output": None},
+ ),
+ ]
+ )
+
+
+class FakeClient:
+ default_headers: dict[str, str] = {}
+ api_key = "test-key"
+
+ def __init__(self):
+ self.responses = FakeResponses()
+
+
+def test_extract_delta_text_handles_dict_deltas_without_stringifying_empty_values():
+ assert _extract_delta_text({"content": "hello"}) == "hello"
+ assert _extract_delta_text({"text": "hello"}) == "hello"
+ assert _extract_delta_text({"content": ""}) == ""
+
+
+async def test_async_response_falls_back_to_stream_when_output_text_is_unavailable():
+ client = FakeClient()
+
+ response_text, response_id, response_obj = await async_response(
+ client,
+ prompt="",
+ model="flow-id",
+ )
+
+ assert response_text == "Hello world"
+ assert response_id == "stream-response"
+ assert response_obj == {"id": "stream-response", "output": None}
+ assert [call["stream"] for call in client.responses.calls] == [False, True]
diff --git a/tests/unit/test_agent_sources.py b/tests/unit/test_agent_sources.py
new file mode 100644
index 000000000..4ed3172c4
--- /dev/null
+++ b/tests/unit/test_agent_sources.py
@@ -0,0 +1,63 @@
+from agent import _extract_retrieval_sources
+
+
+def test_extract_retrieval_sources_from_langflow_tool_use_content_block():
+ response = {
+ "outputs": [
+ {
+ "results": {
+ "message": {
+ "data": {
+ "content_blocks": [
+ {
+ "contents": [
+ {
+ "type": "tool_use",
+ "name": "search_documents",
+ "output": [
+ {
+ "text": "purple elephants dancing",
+ "filename": "sdk_test_doc.md",
+ "mimetype": "text/markdown",
+ "page": 0,
+ }
+ ],
+ }
+ ]
+ }
+ ]
+ }
+ }
+ }
+ }
+ ]
+ }
+
+ assert _extract_retrieval_sources(response) == [
+ {
+ "filename": "sdk_test_doc.md",
+ "text": "purple elephants dancing",
+ "score": 0,
+ "page": 0,
+ "mimetype": "text/markdown",
+ }
+ ]
+
+
+def test_extract_retrieval_sources_ignores_assistant_message_text():
+ response = {
+ "outputs": [
+ {
+ "results": {
+ "message": {
+ "data": {
+ "text": "The document says the animals are purple.",
+ "content_blocks": [],
+ }
+ }
+ }
+ }
+ ]
+ }
+
+ assert _extract_retrieval_sources(response) == []
diff --git a/tests/unit/test_api_key_service.py b/tests/unit/test_api_key_service.py
new file mode 100644
index 000000000..f1de9f058
--- /dev/null
+++ b/tests/unit/test_api_key_service.py
@@ -0,0 +1,94 @@
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+def test_api_key_hash_uses_keyed_digest(monkeypatch):
+ from services.api_key_service import API_KEY_HASH_PREFIX, APIKeyService
+
+ monkeypatch.setattr("config.settings.SESSION_SECRET", "unit-test-session-secret")
+
+ service = APIKeyService()
+ keyed_hash = service._hash_key("orag_test_key")
+
+ assert keyed_hash.startswith(API_KEY_HASH_PREFIX)
+ assert keyed_hash != service._legacy_hash_key("orag_test_key")
+
+
+async def test_create_key_stores_hmac_hash(monkeypatch):
+ from services.api_key_service import API_KEY_HASH_PREFIX, APIKeyService
+
+ monkeypatch.setattr("config.settings.SESSION_SECRET", "unit-test-session-secret")
+ monkeypatch.setattr("secrets.token_urlsafe", lambda n: "deterministic-token")
+
+ opensearch_client = AsyncMock()
+ opensearch_client.index.return_value = {"result": "created"}
+ monkeypatch.setattr("config.settings.clients.opensearch", opensearch_client)
+
+ service = APIKeyService()
+ result = await service.create_key(
+ user_id="user-1",
+ user_email="[email protected]",
+ name="test key",
+ )
+
+ assert result["success"] is True
+ stored_doc = opensearch_client.index.await_args.kwargs["body"]
+ assert stored_doc["key_hash"].startswith(API_KEY_HASH_PREFIX)
+ assert stored_doc["key_hash"] == service._hash_key(result["api_key"])
+ assert stored_doc["key_hash"] != result["api_key"]
+
+
+async def test_validate_key_accepts_and_migrates_legacy_hash(monkeypatch):
+ from services.api_key_service import APIKeyService
+
+ monkeypatch.setattr("config.settings.SESSION_SECRET", "unit-test-session-secret")
+
+ service = APIKeyService()
+ api_key = "orag_legacy_key"
+ legacy_hash = service._legacy_hash_key(api_key)
+ keyed_hash = service._hash_key(api_key)
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search.return_value = {
+ "hits": {
+ "hits": [
+ {
+ "_source": {
+ "key_id": "key-1",
+ "key_hash": legacy_hash,
+ "user_id": "user-1",
+ "user_email": "[email protected]",
+ "name": "legacy key",
+ }
+ }
+ ]
+ }
+ }
+ monkeypatch.setattr("config.settings.clients.opensearch", opensearch_client)
+
+ user_info = await service.validate_key(api_key)
+
+ assert user_info == {
+ "key_id": "key-1",
+ "user_id": "user-1",
+ "user_email": "[email protected]",
+ "name": "legacy key",
+ }
+ terms = opensearch_client.search.await_args.kwargs["body"]["query"]["bool"]["must"][0]["terms"][
+ "key_hash"
+ ]
+ assert terms == [keyed_hash, legacy_hash]
+
+ update_doc = opensearch_client.update.await_args.kwargs["body"]["doc"]
+ assert update_doc["key_hash"] == keyed_hash
+ assert "last_used_at" in update_doc
diff --git a/tests/unit/test_document_delete_by_id.py b/tests/unit/test_document_delete_by_id.py
new file mode 100644
index 000000000..aa6983258
--- /dev/null
+++ b/tests/unit/test_document_delete_by_id.py
@@ -0,0 +1,112 @@
+import pytest
+
+from api.documents import delete_documents_by_filename_core
+
+
+class FakeOpenSearchClient:
+ def __init__(self, owned_hits=None, visible_hits=None):
+ self.owned_hits = owned_hits or []
+ self.visible_hits = visible_hits or []
+ self.search_calls = []
+ self.delete_calls = []
+
+ async def search(self, *, index, body, scroll=None):
+ self.search_calls.append({"index": index, "body": body, "scroll": scroll})
+ query = body["query"]
+ if "bool" in query:
+ hits = self.owned_hits
+ else:
+ hits = self.visible_hits
+ return {"hits": {"hits": hits}}
+
+ async def delete(self, *, index, id, refresh=True):
+ self.delete_calls.append({"index": index, "id": id, "refresh": refresh})
+ return {"result": "deleted"}
+
+
+class FakeSessionManager:
+ def __init__(self, opensearch_client):
+ self.opensearch_client = opensearch_client
+
+ def get_user_opensearch_client(self, user_id, jwt_token):
+ assert user_id == "user-1"
+ assert jwt_token == "jwt-token"
+ return self.opensearch_client
+
+
+async def test_delete_documents_by_filename_deletes_owned_ids_with_backend_client(monkeypatch):
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents")
+ opensearch_client = FakeOpenSearchClient(
+ owned_hits=[
+ {"_id": "chunk-1", "_source": {"owner": "user-1"}},
+ {"_id": "chunk-2", "_source": {"owner": "user-1"}},
+ ]
+ )
+ backend_opensearch_client = FakeOpenSearchClient()
+ monkeypatch.setattr("config.settings.clients.opensearch", backend_opensearch_client)
+
+ payload, status_code = await delete_documents_by_filename_core(
+ filename=" report.pdf ",
+ session_manager=FakeSessionManager(opensearch_client),
+ user_id="user-1",
+ jwt_token="jwt-token",
+ )
+
+ assert status_code == 200
+ assert payload["success"] is True
+ assert payload["deleted_chunks"] == 2
+ assert len(opensearch_client.search_calls) == 1
+ assert opensearch_client.search_calls[0]["body"]["query"] == {
+ "bool": {
+ "filter": [
+ {"term": {"filename": "report.pdf"}},
+ {"term": {"owner": "user-1"}},
+ ]
+ }
+ }
+ assert opensearch_client.delete_calls == []
+ assert backend_opensearch_client.delete_calls == [
+ {"index": "documents", "id": "chunk-1", "refresh": True},
+ {"index": "documents", "id": "chunk-2", "refresh": True},
+ ]
+
+
+async def test_delete_documents_by_filename_denies_visible_non_owner(monkeypatch):
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents")
+ opensearch_client = FakeOpenSearchClient(
+ owned_hits=[],
+ visible_hits=[{"_id": "shared-chunk", "_source": {"owner": "other-user"}}],
+ )
+
+ payload, status_code = await delete_documents_by_filename_core(
+ filename="shared.pdf",
+ session_manager=FakeSessionManager(opensearch_client),
+ user_id="user-1",
+ jwt_token="jwt-token",
+ )
+
+ assert status_code == 403
+ assert payload["success"] is False
+ assert payload["deleted_chunks"] == 0
+ assert "only the document owner" in payload["error"]
+ assert opensearch_client.delete_calls == []
+
+
+async def test_delete_documents_by_filename_returns_404_when_missing(monkeypatch):
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents")
+ opensearch_client = FakeOpenSearchClient(owned_hits=[], visible_hits=[])
+
+ payload, status_code = await delete_documents_by_filename_core(
+ filename="missing.pdf",
+ session_manager=FakeSessionManager(opensearch_client),
+ user_id="user-1",
+ jwt_token="jwt-token",
+ )
+
+ assert status_code == 404
+ assert payload["success"] is False
+ assert payload["deleted_chunks"] == 0
+ assert opensearch_client.delete_calls == []
diff --git a/tests/unit/test_embedding_fields.py b/tests/unit/test_embedding_fields.py
index 79e6ded21..1fa1acbbb 100644
--- a/tests/unit/test_embedding_fields.py
+++ b/tests/unit/test_embedding_fields.py
@@ -8,11 +8,13 @@ OpenRAG's ``knn_vector`` field mapping. Callers across ``config.settings``,
JVector/DiskANN method configuration with only the dimension varying per
embedding model.
"""
-from typing import Any, Dict
+
+from types import SimpleNamespace
+from typing import Any
import pytest
-from utils.embedding_fields import build_knn_vector_field
+from utils.embedding_fields import build_knn_vector_field, get_embedding_field_name
class TestBuildKnnVectorFieldStructure:
@@ -99,6 +101,26 @@ class TestBuildKnnVectorFieldCallSitesMatch:
def test_index_body_uses_helper_output(self) -> None:
from config.settings import INDEX_BODY, VECTOR_DIM
- chunk_field: Dict[str, Any] = INDEX_BODY["mappings"]["properties"]["chunk_embedding"]
+ chunk_field: dict[str, Any] = INDEX_BODY["mappings"]["properties"]["chunk_embedding"]
expected = build_knn_vector_field(VECTOR_DIM)
assert chunk_field == expected
+
+ @pytest.mark.asyncio
+ async def test_create_index_body_precreates_configured_embedding_field(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(
+ "config.settings.get_openrag_config",
+ lambda: SimpleNamespace(
+ knowledge=SimpleNamespace(embedding_model="text-embedding-3-large")
+ ),
+ )
+
+ from utils.embeddings import create_index_body
+
+ body = await create_index_body("text-embedding-3-large", 3072)
+ properties = body["mappings"]["properties"]
+ embedding_field = get_embedding_field_name("text-embedding-3-large")
+
+ assert properties[embedding_field] == build_knn_vector_field(3072)
+ assert properties["owner_email"] == {"type": "keyword"}
diff --git a/tests/unit/test_flow_opensearch_outputs.py b/tests/unit/test_flow_opensearch_outputs.py
new file mode 100644
index 000000000..2f49dc6b3
--- /dev/null
+++ b/tests/unit/test_flow_opensearch_outputs.py
@@ -0,0 +1,78 @@
+import json
+from pathlib import Path
+
+
+def _load_flow(flow_path: str) -> dict:
+ return json.loads(Path(flow_path).read_text(encoding="utf-8"))
+
+
+def _opensearch_nodes(flow: dict) -> list[dict]:
+ return [
+ node
+ for node in flow["data"]["nodes"]
+ if "OpenSearchVectorStoreComponent" in node.get("data", {}).get("type", "")
+ ]
+
+
+def test_opensearch_component_returns_table_search_results():
+ code = Path("flows/components/opensearch_multimodal.py").read_text(encoding="utf-8")
+
+ assert "from lfx.schema.dataframe import Table" in code
+ assert "def search_documents(self) -> Table:" in code
+ assert "return Table(data=raw_list)" in code
+ assert 'name="dataframe"' not in code
+ assert "def as_dataframe" not in code
+
+
+def test_embedded_opensearch_nodes_expose_table_search_results():
+ for flow_path in (
+ "flows/ingestion_flow.json",
+ "flows/openrag_nudges.json",
+ "flows/openrag_url_mcp.json",
+ ):
+ flow = _load_flow(flow_path)
+ for node in _opensearch_nodes(flow):
+ outputs = node["data"]["node"]["outputs"]
+ if any(output.get("name") == "component_as_tool" for output in outputs):
+ continue
+
+ search_output = next(
+ output for output in outputs if output.get("name") == "search_results"
+ )
+ assert search_output["method"] == "search_documents"
+ assert search_output["types"] == ["Table"]
+ assert all(output.get("name") != "dataframe" for output in outputs)
+
+
+def test_nudges_flow_uses_opensearch_search_results_table():
+ flow = _load_flow("flows/openrag_nudges.json")
+ opensearch_node_ids = {node["id"] for node in _opensearch_nodes(flow)}
+ parser_node_ids = {
+ node["id"]
+ for node in flow["data"]["nodes"]
+ if node.get("data", {}).get("type") == "ParserComponent"
+ }
+
+ opensearch_to_parser_edges = [
+ edge
+ for edge in flow["data"]["edges"]
+ if edge.get("source") in opensearch_node_ids and edge.get("target") in parser_node_ids
+ ]
+
+ assert len(opensearch_to_parser_edges) == 1
+ source_handle = opensearch_to_parser_edges[0]["data"]["sourceHandle"]
+ assert source_handle["name"] == "search_results"
+ assert source_handle["output_types"] == ["Table"]
+
+
+def test_parser_components_are_not_forked_for_list_data():
+ for flow_path in ("flows/openrag_nudges.json", "flows/openrag_url_mcp.json"):
+ flow = _load_flow(flow_path)
+ parser_node = next(
+ node
+ for node in flow["data"]["nodes"]
+ if node.get("data", {}).get("type") == "ParserComponent"
+ )
+ code = parser_node["data"]["node"]["template"]["code"]["value"]
+
+ assert "return DataFrame(data=[item.data for item in input_data]), None" not in code
diff --git a/tests/unit/test_group_acl.py b/tests/unit/test_group_acl.py
new file mode 100644
index 000000000..932f54eb8
--- /dev/null
+++ b/tests/unit/test_group_acl.py
@@ -0,0 +1,819 @@
+import asyncio
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+import jwt
+import pytest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[2]
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+def test_canonical_group_role_is_compact_and_provider_scoped():
+ from utils.group_acl import canonical_group_role
+
+ role = canonical_group_role(
+ "m365",
+ "00000000-0000-0000-0000-000000000001",
+ "00000000-0000-0000-0000-000000000002",
+ )
+
+ assert role == "g:m365:AAAAAAAAAAAAAAAAAAAAAQ:AAAAAAAAAAAAAAAAAAAAAg"
+
+
+def test_canonical_user_principal_is_compact_and_provider_scoped():
+ from utils.group_acl import canonical_user_principal
+
+ principal = canonical_user_principal(
+ "gdrive",
+ "example.com",
+ )
+
+ assert principal.startswith("u:gdrive:")
+ assert principal == canonical_user_principal(
+ "gdrive",
+ "EXAMPLE.COM",
+ )
+
+
+def test_opensearch_jwt_does_not_include_connector_group_roles(monkeypatch):
+ from session_manager import SessionManager, User
+
+ monkeypatch.setenv("JWT_SIGNING_KEY", "unit-test-secret-with-32-bytes!!")
+
+ manager = SessionManager("test")
+ user = User(user_id="user-1", email="[email protected]", name="User")
+
+ token = manager.create_opensearch_jwt_token(user, ttl_seconds=60)
+ payload = jwt.decode(
+ token.removeprefix("Bearer "),
+ "unit-test-secret-with-32-bytes!!",
+ algorithms=["HS256"],
+ audience=["opensearch", "openrag"],
+ )
+
+ assert payload["roles"] == ["openrag_user"]
+ assert payload["sub"] == "user-1"
+
+
+def test_opensearch_jwt_default_ttl_tracks_ingestion_timeout(monkeypatch):
+ from session_manager import SessionManager, User
+
+ monkeypatch.setenv("JWT_SIGNING_KEY", "unit-test-secret-with-32-bytes!!")
+ monkeypatch.delenv("OPENRAG_OPENSEARCH_JWT_TTL", raising=False)
+ monkeypatch.setattr("config.settings.INGESTION_TIMEOUT", 3600)
+
+ manager = SessionManager("test")
+ user = User(user_id="user-1", email="[email protected]", name="User")
+
+ token = manager.create_opensearch_jwt_token(user)
+ payload = jwt.decode(
+ token.removeprefix("Bearer "),
+ "unit-test-secret-with-32-bytes!!",
+ algorithms=["HS256"],
+ audience=["opensearch", "openrag"],
+ )
+
+ assert payload["exp"] - payload["iat"] == 3900
+
+
+async def test_group_acl_service_uses_connector_hooks_generically():
+ from services.group_acl_service import GroupACLService
+ from session_manager import User
+
+ @dataclass
+ class Connection:
+ connection_id: str
+ connector_type: str
+ is_active: bool = True
+
+ class ConnectionManager:
+ async def list_connections(self, user_id=None):
+ assert user_id == "user-1"
+ return [
+ Connection("sharepoint-1", "sharepoint"),
+ Connection("custom-1", "custom"),
+ ]
+
+ class Connector:
+ def __init__(self, roles):
+ self.roles = roles
+
+ async def get_current_user_group_roles(self):
+ return self.roles
+
+ class ConnectorService:
+ connection_manager = ConnectionManager()
+
+ async def get_connector(self, connection_id):
+ return {
+ "sharepoint-1": Connector(["g:m365:t:g1", "g:m365:t:g2"]),
+ "custom-1": Connector(["g:custom:t:g2", "g:m365:t:g1"]),
+ }[connection_id]
+
+ service = GroupACLService(ConnectorService(), cache_ttl_seconds=0)
+ roles = await service.get_user_group_roles(
+ User(user_id="user-1", email="[email protected]", name="User")
+ )
+
+ assert roles == ["g:m365:t:g1", "g:m365:t:g2", "g:custom:t:g2"]
+ assert service._cache == {}
+ assert service._locks == {}
+
+
+async def test_group_acl_service_disabled_cache_does_not_retain_locks():
+ from services.group_acl_service import GroupACLService
+ from session_manager import User
+
+ @dataclass
+ class Connection:
+ connection_id: str
+ connector_type: str
+ is_active: bool = True
+
+ class ConnectionManager:
+ async def list_connections(self, user_id=None):
+ return [Connection(f"{user_id}-connection", "custom")]
+
+ class Connector:
+ async def get_current_user_group_roles(self):
+ return ["g:custom:t:g1"]
+
+ class ConnectorService:
+ connection_manager = ConnectionManager()
+
+ async def get_connector(self, connection_id):
+ return Connector()
+
+ service = GroupACLService(ConnectorService(), cache_ttl_seconds=0)
+
+ for user_id in ("user-1", "user-2"):
+ roles = await service.get_user_group_roles(User(user_id=user_id, email=None, name=None))
+ assert roles == ["g:custom:t:g1"]
+
+ assert service._cache == {}
+ assert service._locks == {}
+
+
+def test_group_acl_service_invalidation_drops_cache_and_locks():
+ from services.group_acl_service import GroupACLService
+
+ service = GroupACLService(connector_service=object(), cache_ttl_seconds=60)
+ service._cache["user-1"] = (999999.0, ["g:test:t:g1"])
+ service._locks["user-1"] = asyncio.Lock()
+ service._cache["user-2"] = (999999.0, ["g:test:t:g2"])
+ service._locks["user-2"] = asyncio.Lock()
+
+ service.invalidate_user("user-1")
+
+ assert "user-1" not in service._cache
+ assert "user-1" not in service._locks
+ assert "user-2" in service._cache
+ assert "user-2" in service._locks
+
+ service.clear()
+
+ assert service._cache == {}
+ assert service._locks == {}
+
+
+def test_security_roles_include_acl_dls_queries():
+ for rel_path in ("securityconfig/roles.yml", "cloud_securityconfig/roles.yml"):
+ roles = yaml.safe_load((ROOT / rel_path).read_text())
+ index_permissions = roles["openrag_user_role"]["index_permissions"]
+ cluster_permissions = roles["openrag_user_role"]["cluster_permissions"]
+ assert "indices:data/write/bulk" not in cluster_permissions
+ assert "indices:data/write/index" not in cluster_permissions
+ assert not any("alerting" in permission for permission in cluster_permissions)
+
+ document_permission = index_permissions[0]
+ document_actions = document_permission["allowed_actions"]
+ assert "read" in document_actions
+ assert "crud" not in document_actions
+ assert "indices:data/write/index" not in document_actions
+ assert "indices:data/write/update/byquery" not in document_actions
+ assert "indices:admin/mappings/put" not in document_actions
+
+ dls = index_permissions[0]["dls"]
+ assert '{"term":{"owner":"${user.name}"}}' in dls
+ assert '{"term":{"owner":"${attr.jwt.email}"}}' in dls
+ assert '{"term":{"allowed_users":"${user.name}"}}' in dls
+ assert '{"term":{"allowed_users":"${attr.jwt.email}"}}' in dls
+ assert '{"terms":{"allowed_groups":[${user.roles}]}}' not in dls
+ assert (
+ '{"terms":{"allowed_principals":{"index":"openrag_dls_principals",'
+ '"id":"${user.name}","path":"principals"}}}' in dls
+ )
+ principal_permission = next(
+ permission
+ for permission in index_permissions
+ if "openrag_dls_principals" in permission["index_patterns"]
+ )
+ assert "crud" not in principal_permission["allowed_actions"]
+ assert "indices:data/write/index" not in principal_permission["allowed_actions"]
+ assert principal_permission["dls"] == '{"term":{"user_name":"${user.name}"}}\n'
+
+
+async def test_dls_principal_service_writes_user_lookup_rows():
+ from services.dls_principal_service import DLSPrincipalService
+ from session_manager import User
+
+ @dataclass
+ class Connection:
+ connection_id: str
+ connector_type: str
+ is_active: bool = True
+
+ class ConnectionManager:
+ async def list_connections(self, user_id=None):
+ assert user_id == "user-1"
+ return [
+ Connection("drive-1", "google_drive"),
+ Connection("inactive-1", "google_drive", is_active=False),
+ ]
+
+ def get_auth_user_principals(self, user):
+ assert user.email == "[email protected]"
+ return ["u:gdrive:example.com:user"]
+
+ class Connector:
+ async def get_current_user_principals(self):
+ return ["u:gdrive:t:user", "u:gdrive:t:user"]
+
+ async def get_current_user_group_roles(self):
+ return ["g:gdrive:t:engineering"]
+
+ async def get_current_user_principal_labels(self):
+ return [
+ {
+ "principal": "g:gdrive:t:engineering",
+ "kind": "group",
+ "provider": "gdrive",
+ "display_name": "Engineering",
+ }
+ ]
+
+ class ConnectorService:
+ connection_manager = ConnectionManager()
+
+ async def get_connector(self, connection_id):
+ assert connection_id == "drive-1"
+ return Connector()
+
+ class Indices:
+ async def exists(self, index):
+ assert index == "openrag_dls_principals"
+ return False
+
+ async def create(self, index, body):
+ assert index == "openrag_dls_principals"
+ assert body["mappings"]["properties"]["principals"]["type"] == "keyword"
+
+ class OpenSearchClient:
+ indices = Indices()
+
+ def __init__(self):
+ self.index_calls = []
+
+ async def index(self, **kwargs):
+ self.index_calls.append(kwargs)
+
+ opensearch_client = OpenSearchClient()
+ service = DLSPrincipalService(ConnectorService(), opensearch_client=opensearch_client)
+
+ principals = await service.refresh_user_principals(
+ User(
+ user_id="user-1",
+ name="User",
+ provider="google",
+ opensearch_username="ibmlhapikey_user-1",
+ ),
+ group_roles=["g:gdrive:t:engineering"],
+ )
+
+ assert principals[0] == "g:gdrive:t:engineering"
+ assert "u:gdrive:t:user" in principals
+ assert "u:gdrive:example.com:user" in principals
+ assert len(opensearch_client.index_calls) == 2
+ assert {call["id"] for call in opensearch_client.index_calls} == {
+ "ibmlhapikey_user-1",
+ "user-1",
+ }
+ body = opensearch_client.index_calls[0]["body"]
+ assert any(
+ label.get("principal") == "g:gdrive:t:engineering"
+ and label.get("kind") == "group"
+ and label.get("provider") == "gdrive"
+ and label.get("display_name") == "Engineering"
+ for label in body["principal_labels"]
+ )
+ assert any(
+ label.get("principal") == "u:gdrive:example.com:user"
+ and label.get("kind") == "user"
+ and label.get("provider") == "gdrive"
+ and label.get("display_name") == "User"
+ and label.get("email") == "[email protected]"
+ and label.get("external_id") == "user-1"
+ for label in body["principal_labels"]
+ )
+ for call in opensearch_client.index_calls:
+ assert call["index"] == "openrag_dls_principals"
+ assert call["refresh"] == "wait_for"
+ assert call["body"]["principals"] == principals
+
+
+async def test_dls_principal_service_caches_and_coalesces_lookup_refreshes():
+ from services.dls_principal_service import DLSPrincipalService
+ from session_manager import User
+
+ @dataclass
+ class Connection:
+ connection_id: str
+ connector_type: str
+ is_active: bool = True
+
+ class ConnectionManager:
+ def __init__(self):
+ self.list_calls = 0
+
+ async def list_connections(self, user_id=None):
+ assert user_id == "user-1"
+ self.list_calls += 1
+ await asyncio.sleep(0.01)
+ return [Connection("drive-1", "google_drive")]
+
+ def get_auth_user_principals(self, user):
+ assert user.email == "[email protected]"
+ return ["u:gdrive:example.com:user"]
+
+ class Connector:
+ async def get_current_user_principals(self):
+ return ["u:gdrive:t:user"]
+
+ async def get_current_user_group_roles(self):
+ return ["g:gdrive:t:engineering"]
+
+ class ConnectorService:
+ def __init__(self, connection_manager):
+ self.connection_manager = connection_manager
+
+ async def get_connector(self, connection_id):
+ assert connection_id == "drive-1"
+ return Connector()
+
+ class Indices:
+ async def exists(self, index):
+ assert index == "openrag_dls_principals"
+ return True
+
+ class OpenSearchClient:
+ indices = Indices()
+
+ def __init__(self):
+ self.index_calls = []
+
+ async def index(self, **kwargs):
+ self.index_calls.append(kwargs)
+
+ connection_manager = ConnectionManager()
+ opensearch_client = OpenSearchClient()
+ service = DLSPrincipalService(
+ ConnectorService(connection_manager),
+ opensearch_client=opensearch_client,
+ refresh_ttl_seconds=60,
+ )
+ user = User(
+ user_id="user-1",
+ name="User",
+ provider="google",
+ opensearch_username="ibmlhapikey_user-1",
+ )
+
+ first, second = await asyncio.gather(
+ service.refresh_user_principals(user),
+ service.refresh_user_principals(user),
+ )
+ third = await service.refresh_user_principals(user)
+
+ assert first == second == third
+ assert "g:gdrive:t:engineering" in first
+ assert connection_manager.list_calls == 1
+ assert len(opensearch_client.index_calls) == 2
+ assert {call["id"] for call in opensearch_client.index_calls} == {
+ "ibmlhapikey_user-1",
+ "user-1",
+ }
+
+
+async def test_dls_principal_service_disabled_refresh_cache_does_not_retain_locks():
+ from services.dls_principal_service import DLSPrincipalService
+ from session_manager import User
+
+ class ConnectionManager:
+ async def list_connections(self, user_id=None):
+ return []
+
+ def get_auth_user_principals(self, user):
+ return [f"u:test:{user.user_id}"]
+
+ class ConnectorService:
+ connection_manager = ConnectionManager()
+
+ class Indices:
+ async def exists(self, index):
+ return True
+
+ class OpenSearchClient:
+ indices = Indices()
+
+ def __init__(self):
+ self.index_calls = []
+
+ async def index(self, **kwargs):
+ self.index_calls.append(kwargs)
+
+ opensearch_client = OpenSearchClient()
+ service = DLSPrincipalService(
+ ConnectorService(),
+ opensearch_client=opensearch_client,
+ refresh_ttl_seconds=0,
+ )
+
+ for user_id in ("user-1", "user-2"):
+ principals = await service.refresh_user_principals(
+ User(user_id=user_id, email=None, name=None, provider="test")
+ )
+ assert principals == [f"u:test:{user_id}"]
+
+ assert len(opensearch_client.index_calls) == 2
+ assert service._cache == {}
+ assert service._locks == {}
+
+
+async def test_dls_principal_service_skips_connector_lookup_without_opensearch_client(monkeypatch):
+ from services.dls_principal_service import DLSPrincipalService
+ from session_manager import User
+
+ monkeypatch.setattr("config.settings.IBM_AUTH_ENABLED", True)
+ monkeypatch.setattr("config.settings.OPENSEARCH_PASSWORD", "")
+
+ class ConnectionManager:
+ async def list_connections(self, user_id=None):
+ raise AssertionError("connector lookup should not run without an OpenSearch client")
+
+ class ConnectorService:
+ connection_manager = ConnectionManager()
+
+ service = DLSPrincipalService(
+ ConnectorService(),
+ opensearch_client=None,
+ refresh_ttl_seconds=60,
+ )
+
+ principals = await service.refresh_user_principals(
+ User(
+ user_id="user-1",
+ name="User",
+ provider="google",
+ )
+ )
+
+ assert principals == []
+
+
+def test_dls_principal_service_uses_basic_admin_client_in_ibm(monkeypatch):
+ from services.dls_principal_service import DLSPrincipalService
+
+ class Clients:
+ def __init__(self):
+ self.calls = []
+ self.opensearch = object()
+
+ def create_basic_opensearch_client(self, username, password):
+ self.calls.append((username, password))
+ return "admin-client"
+
+ clients = Clients()
+ monkeypatch.setattr("config.settings.IBM_AUTH_ENABLED", True)
+ monkeypatch.setattr("config.settings.OPENSEARCH_USERNAME", "admin-user")
+ monkeypatch.setattr("config.settings.OPENSEARCH_PASSWORD", "admin-pass")
+ monkeypatch.setattr("config.settings.clients", clients)
+
+ service = DLSPrincipalService(connector_service=None)
+
+ assert service._get_opensearch_client() == "admin-client"
+ assert service._get_opensearch_client() == "admin-client"
+ assert clients.calls == [("admin-user", "admin-pass")]
+
+
+async def test_opensearch_init_adds_missing_acl_keyword_mappings():
+ from utils.opensearch_init import _ensure_keyword_mappings
+
+ class Indices:
+ def __init__(self):
+ self.put_mapping_calls = []
+
+ async def get_mapping(self, index):
+ assert index == "documents"
+ return {
+ "documents": {
+ "mappings": {
+ "properties": {
+ "allowed_users": {"type": "keyword"},
+ "allowed_groups": {"type": "keyword"},
+ }
+ }
+ }
+ }
+
+ async def put_mapping(self, **kwargs):
+ self.put_mapping_calls.append(kwargs)
+
+ class OpenSearchClient:
+ def __init__(self):
+ self.indices = Indices()
+
+ opensearch_client = OpenSearchClient()
+
+ await _ensure_keyword_mappings(
+ opensearch_client,
+ "documents",
+ ["allowed_users", "allowed_groups", "allowed_principals"],
+ )
+
+ assert opensearch_client.indices.put_mapping_calls == [
+ {
+ "index": "documents",
+ "body": {"properties": {"allowed_principals": {"type": "keyword"}}},
+ }
+ ]
+
+
+async def test_connector_service_mints_plain_jwt_when_session_user_is_missing(
+ monkeypatch,
+):
+ from session_manager import SessionManager
+
+ monkeypatch.setenv("JWT_SIGNING_KEY", "unit-test-secret-with-32-bytes!!")
+ monkeypatch.setattr("config.settings.IBM_AUTH_ENABLED", False)
+
+ @dataclass
+ class Connection:
+ connection_id: str
+ connector_type: str
+ is_active: bool = True
+
+ class Connector:
+ async def get_current_user_group_roles(self):
+ return ["g:test:t:g1"]
+
+ class ConnectionManager:
+ async def list_connections(self, user_id=None):
+ assert user_id == "stored-user-id"
+ return [Connection("connection-1", "custom")]
+
+ async def get_connector(self, connection_id):
+ assert connection_id == "connection-1"
+ return Connector()
+
+ session_manager = SessionManager("test")
+ from connectors.service import ConnectorService
+
+ service = ConnectorService(
+ patched_async_client=None,
+ embed_model="test",
+ index_name="test-index",
+ session_manager=session_manager,
+ )
+
+ service.connection_manager = ConnectionManager()
+
+ token = await service._get_effective_sync_jwt("stored-user-id")
+ payload = jwt.decode(
+ token.removeprefix("Bearer "),
+ "unit-test-secret-with-32-bytes!!",
+ algorithms=["HS256"],
+ audience=["opensearch", "openrag"],
+ )
+
+ assert payload["sub"] == "stored-user-id"
+ assert payload["roles"] == ["openrag_user"]
+
+
+def test_google_drive_file_acl_group_is_canonicalized(tmp_path):
+ from connectors.google_drive.connector import GoogleDriveConnector
+ from connectors.google_drive_acl import google_drive_group_role, google_drive_user_principal
+
+ class Execute:
+ def execute(self):
+ return {
+ "permissions": [
+ {
+ "type": "group",
+ "role": "reader",
+ "emailAddress": "[email protected]",
+ },
+ {
+ "type": "user",
+ "role": "owner",
+ "emailAddress": "[email protected]",
+ },
+ ]
+ }
+
+ class Permissions:
+ def list(self, **kwargs):
+ assert kwargs["fileId"] == "file-1"
+ return Execute()
+
+ class Service:
+ def permissions(self):
+ return Permissions()
+
+ connector = GoogleDriveConnector(
+ {
+ "client_id": "client",
+ "client_secret": "secret",
+ "token_file": str(tmp_path / "token.json"),
+ }
+ )
+ connector.service = Service()
+
+ acl = connector._extract_google_drive_acl({"id": "file-1"})
+
+ assert acl.owner == "[email protected]"
+ assert acl.allowed_users == ["[email protected]"]
+ assert acl.allowed_groups == [google_drive_group_role("[email protected]")]
+ assert acl.allowed_principals == [
+ google_drive_group_role("[email protected]"),
+ google_drive_user_principal("[email protected]"),
+ ]
+ assert acl.allowed_principal_labels == [
+ {
+ "principal": google_drive_group_role("[email protected]"),
+ "kind": "group",
+ "provider": "gdrive",
+ "display_name": "[email protected]",
+ "email": "[email protected]",
+ "external_id": "[email protected]",
+ },
+ {
+ "principal": google_drive_user_principal("[email protected]"),
+ "kind": "user",
+ "provider": "gdrive",
+ "display_name": "[email protected]",
+ "email": "[email protected]",
+ "external_id": "[email protected]",
+ },
+ ]
+
+
+def test_connection_manager_resolves_auth_user_principals(tmp_path):
+ from connectors.connection_manager import ConnectionManager
+ from connectors.google_drive_acl import google_drive_user_principal
+ from session_manager import User
+
+ manager = ConnectionManager(connections_file=str(tmp_path / "connections.json"))
+
+ assert manager.get_auth_user_principals(
+ User(
+ user_id="google-subject",
+ name="User",
+ provider="google",
+ )
+ ) == [google_drive_user_principal("[email protected]")]
+
+
+async def test_google_drive_group_roles_use_directory_groups(monkeypatch):
+ from connectors.google_drive_acl import (
+ get_current_user_google_group_roles,
+ google_drive_group_role,
+ )
+
+ class Credentials:
+ id_token = jwt.encode(
+ {"email": "[email protected]"},
+ "google-test-secret-with-32-bytes",
+ algorithm="HS256",
+ )
+
+ class Execute:
+ def __init__(self, response):
+ self.response = response
+
+ def execute(self):
+ return self.response
+
+ class Groups:
+ def list(self, **kwargs):
+ assert kwargs["userKey"] == "[email protected]"
+ assert kwargs["domain"] == "example.com"
+ assert "customer" not in kwargs
+ return Execute(
+ {
+ "groups": [
+ {"email": "[email protected]"},
+ {"email": "[email protected]"},
+ ]
+ }
+ )
+
+ class DirectoryService:
+ def groups(self):
+ return Groups()
+
+ def fake_build(*args, **kwargs):
+ if args[:2] == ("cloudidentity", "v1"):
+ raise RuntimeError("Cloud Identity unavailable")
+ assert args[:2] == ("admin", "directory_v1")
+ return DirectoryService()
+
+ monkeypatch.setattr("connectors.google_drive_acl.build", fake_build)
+
+ roles = await get_current_user_google_group_roles(
+ drive_service=None,
+ credentials=Credentials(),
+ )
+
+ assert roles == [
+ google_drive_group_role("[email protected]"),
+ google_drive_group_role("[email protected]"),
+ ]
+
+
+async def test_google_drive_group_roles_prefer_cloud_identity(monkeypatch):
+ from connectors.google_drive_acl import (
+ get_current_user_google_group_roles,
+ google_drive_group_role,
+ )
+
+ class Credentials:
+ id_token = jwt.encode(
+ {"email": "[email protected]"},
+ "google-test-secret-with-32-bytes",
+ algorithm="HS256",
+ )
+
+ class Execute:
+ def __init__(self, response):
+ self.response = response
+
+ def execute(self):
+ return self.response
+
+ class Memberships:
+ def searchTransitiveGroups(self, **kwargs):
+ assert kwargs["parent"] == "groups/-"
+ assert "member_key_id == '[email protected]'" in kwargs["query"]
+ return Execute(
+ {
+ "memberships": [
+ {"groupKey": {"id": "[email protected]"}},
+ {"groupKey": {"id": "[email protected]"}},
+ ]
+ }
+ )
+
+ class Groups:
+ def memberships(self):
+ return Memberships()
+
+ class CloudIdentityService:
+ def groups(self):
+ return Groups()
+
+ def fake_build(*args, **kwargs):
+ assert args[:2] == ("cloudidentity", "v1")
+ return CloudIdentityService()
+
+ monkeypatch.setattr("connectors.google_drive_acl.build", fake_build)
+
+ roles = await get_current_user_google_group_roles(
+ drive_service=None,
+ credentials=Credentials(),
+ )
+
+ assert roles == [
+ google_drive_group_role("[email protected]"),
+ google_drive_group_role("[email protected]"),
+ ]
diff --git a/tests/unit/test_knowledge_filter_service.py b/tests/unit/test_knowledge_filter_service.py
new file mode 100644
index 000000000..5e54dc154
--- /dev/null
+++ b/tests/unit/test_knowledge_filter_service.py
@@ -0,0 +1,108 @@
+from types import SimpleNamespace
+
+import pytest
+
+from services.knowledge_filter_service import (
+ KNOWLEDGE_FILTERS_INDEX_NAME,
+ KnowledgeFilterService,
+)
+
+
+class _Indices:
+ async def refresh(self, index):
+ return {"acknowledged": True, "index": index}
+
+
+async def test_knowledge_filter_writes_use_admin_client_after_user_visibility_check(
+ monkeypatch,
+):
+ user_client = SimpleNamespace(get_calls=[], write_calls=[])
+ admin_client = SimpleNamespace(
+ index_calls=[],
+ update_calls=[],
+ delete_calls=[],
+ indices=_Indices(),
+ )
+
+ filter_doc = {
+ "id": "filter-1",
+ "name": "Test filter",
+ "owner": "user-1",
+ "query_data": "{}",
+ }
+ stored_doc = dict(filter_doc)
+
+ async def get(*, index, id):
+ user_client.get_calls.append({"index": index, "id": id})
+ return {"found": True, "_source": dict(stored_doc)}
+
+ async def user_index(**kwargs):
+ user_client.write_calls.append(("index", kwargs))
+
+ async def user_update(**kwargs):
+ user_client.write_calls.append(("update", kwargs))
+
+ async def user_delete(**kwargs):
+ user_client.write_calls.append(("delete", kwargs))
+
+ async def admin_index(**kwargs):
+ admin_client.index_calls.append(kwargs)
+ stored_doc.update(kwargs["body"])
+ return {"result": "created"}
+
+ async def admin_update(**kwargs):
+ admin_client.update_calls.append(kwargs)
+ stored_doc.update(kwargs["body"]["doc"])
+ return {"result": "updated"}
+
+ async def admin_delete(**kwargs):
+ admin_client.delete_calls.append(kwargs)
+ return {"result": "deleted"}
+
+ user_client.get = get
+ user_client.index = user_index
+ user_client.update = user_update
+ user_client.delete = user_delete
+ admin_client.index = admin_index
+ admin_client.update = admin_update
+ admin_client.delete = admin_delete
+
+ class SessionManager:
+ def get_user_opensearch_client(self, user_id, jwt_token):
+ assert user_id == "user-1"
+ assert jwt_token == "Bearer user-token"
+ return user_client
+
+ monkeypatch.setattr(
+ "config.settings.clients",
+ SimpleNamespace(opensearch=admin_client),
+ )
+
+ service = KnowledgeFilterService(SessionManager())
+
+ created = await service.create_knowledge_filter(
+ filter_doc, user_id="user-1", jwt_token="Bearer user-token"
+ )
+ updated = await service.update_knowledge_filter(
+ "filter-1",
+ {"description": "Updated"},
+ user_id="user-1",
+ jwt_token="Bearer user-token",
+ )
+ deleted = await service.delete_knowledge_filter(
+ "filter-1", user_id="user-1", jwt_token="Bearer user-token"
+ )
+
+ assert created["success"] is True
+ assert updated["success"] is True
+ assert deleted["success"] is True
+ assert admin_client.index_calls[0]["index"] == KNOWLEDGE_FILTERS_INDEX_NAME
+ assert admin_client.update_calls[0]["index"] == KNOWLEDGE_FILTERS_INDEX_NAME
+ assert admin_client.delete_calls[0]["index"] == KNOWLEDGE_FILTERS_INDEX_NAME
+ assert user_client.write_calls == []
+ assert user_client.get_calls == [
+ {"index": KNOWLEDGE_FILTERS_INDEX_NAME, "id": "filter-1"},
+ {"index": KNOWLEDGE_FILTERS_INDEX_NAME, "id": "filter-1"},
+ {"index": KNOWLEDGE_FILTERS_INDEX_NAME, "id": "filter-1"},
+ ]
diff --git a/tests/unit/test_langflow_file_service_two_phase.py b/tests/unit/test_langflow_file_service_two_phase.py
index 5e2a6217e..f27c29a07 100644
--- a/tests/unit/test_langflow_file_service_two_phase.py
+++ b/tests/unit/test_langflow_file_service_two_phase.py
@@ -6,8 +6,10 @@ ingestion flow until Docling reports SUCCESS, and must NEVER invoke it when
Docling fails / expires / times out.
"""
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
import pytest
-from unittest.mock import AsyncMock, patch
from models.tasks import (
DoclingPhaseStatus,
@@ -50,7 +52,7 @@ def langflow_service(mock_docling_service):
@pytest.mark.asyncio
async def test_two_phase_success_invokes_langflow_with_task_id(
- langflow_service, mock_polling_service, file_tuple, file_task
+ langflow_service, mock_docling_service, mock_polling_service, file_tuple, file_task
):
mock_polling_service.poll_until_ready.return_value = DoclingPollResult(
outcome=PollOutcome.SUCCESS, elapsed_seconds=2.5
@@ -60,6 +62,15 @@ async def test_two_phase_success_invokes_langflow_with_task_id(
file_tuple=file_tuple,
docling_polling_service=mock_polling_service,
file_task=file_task,
+ owner="owner-123",
+ jwt_token="Bearer jwt-token",
+ )
+
+ mock_docling_service.upload_to_docling_direct_async.assert_awaited_once_with(
+ "test.pdf",
+ b"PDFDATA",
+ user_id="owner-123",
+ auth_header="Bearer jwt-token",
)
# Langflow was invoked exactly once, with the docling_task_id forwarded.
@@ -221,6 +232,37 @@ def test_processor_accepts_injected_polling_service():
assert processor.docling_polling_service is injected
+async def test_langflow_preflight_detects_embedding_dimensions_with_probe(monkeypatch):
+ class FakeEmbeddings:
+ def __init__(self):
+ self.calls = []
+
+ async def create(self, model, input):
+ self.calls.append((model, input))
+ return SimpleNamespace(data=[SimpleNamespace(embedding=[0.1] * 7)])
+
+ fake_embeddings = FakeEmbeddings()
+ fake_client = SimpleNamespace(embeddings=fake_embeddings)
+
+ async def fake_get_litellm_model_name(self, model_name, provider=None, strict=False):
+ assert model_name == "provider/model"
+ assert provider == "provider"
+ return "provider/provider/model"
+
+ monkeypatch.setattr("config.settings.clients._patched_async_client", fake_client)
+ monkeypatch.setattr(
+ "services.models_service.ModelsService.get_litellm_model_name",
+ fake_get_litellm_model_name,
+ )
+
+ svc = LangflowFileService(docling_service=AsyncMock())
+
+ assert await svc._detect_embedding_dimensions("provider/model", "provider") == 7
+ assert await svc._detect_embedding_dimensions("provider/model", "provider") == 7
+ assert fake_embeddings.calls == [("provider/provider/model", ["dimension probe"])]
+
+
@pytest.mark.asyncio
async def test_task_service_threads_polling_service_to_processor(monkeypatch):
"""TaskService.create_langflow_upload_task must forward its injected
diff --git a/tests/unit/test_langflow_ingest_callback.py b/tests/unit/test_langflow_ingest_callback.py
new file mode 100644
index 000000000..4f17d4081
--- /dev/null
+++ b/tests/unit/test_langflow_ingest_callback.py
@@ -0,0 +1,385 @@
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+from fastapi import HTTPException
+
+from api.langflow_ingest import LangflowIngestBatch, LangflowIngestChunk, ingest_langflow_chunks
+from services.document_index_writer import DocumentIndexContext
+from services.langflow_file_service import LangflowFileService
+from services.langflow_ingest_token_service import LangflowIngestTokenService
+
+CALLBACK_GLOBAL_VARS = {
+ "OPENRAG_INGEST_URL",
+ "OPENRAG_INGEST_TOKEN",
+ "OPENRAG_INGEST_RUN_ID",
+ "OPENRAG_INGEST_BATCH_SIZE",
+}
+
+
+async def test_langflow_ingest_callback_indexes_authoritative_token_context():
+ token_service = LangflowIngestTokenService(secret="test-secret" * 4, ttl_seconds=60)
+ context = DocumentIndexContext(
+ document_id="doc-1",
+ filename="source.pdf",
+ mimetype="application/pdf",
+ embedding_model="text-embedding-3-small",
+ owner="user-1",
+ allowed_users=["[email protected]"],
+ allowed_principals=["u:ms:tenant:user"],
+ allowed_principal_labels=[
+ {
+ "principal": "u:ms:tenant:user",
+ "kind": "user",
+ "provider": "ms",
+ "display_name": "User",
+ "email": "[email protected]",
+ }
+ ],
+ ingest_run_id="run-1",
+ )
+ token = token_service.create_token(context)
+
+ class Writer:
+ def __init__(self):
+ self.calls = []
+
+ async def index_chunks(self, context, chunks, *, final=False):
+ self.calls.append((context, chunks, final))
+ return {"indexed_chunks": len(chunks), "document_id": context.document_id}
+
+ writer = Writer()
+ body = LangflowIngestBatch(
+ ingest_run_id="run-1",
+ batch_id=1,
+ final=True,
+ chunks=[
+ LangflowIngestChunk(
+ id="doc-1_0",
+ text="hello",
+ vector=[0.1, 0.2],
+ page=3,
+ metadata={"owner": "forged-owner", "filename": "forged.pdf"},
+ )
+ ],
+ )
+
+ result = await ingest_langflow_chunks(
+ body,
+ authorization=f"Bearer {token}",
+ x_openrag_ingest_token=None,
+ token_service=token_service,
+ writer=writer,
+ )
+
+ indexed_context, chunks, final = writer.calls[0]
+ assert result["status"] == "ok"
+ assert indexed_context.owner == "user-1"
+ assert indexed_context.allowed_users == ["[email protected]"]
+ assert indexed_context.allowed_principals == ["u:ms:tenant:user"]
+ assert indexed_context.allowed_principal_labels == [
+ {
+ "principal": "u:ms:tenant:user",
+ "kind": "user",
+ "provider": "ms",
+ "display_name": "User",
+ "email": "[email protected]",
+ }
+ ]
+ assert chunks[0].chunk_id == "doc-1_1_0"
+ assert chunks[0].metadata["langflow_chunk_id"] == "doc-1_0"
+ assert chunks[0].metadata["owner"] == "forged-owner"
+ assert final is True
+
+ with pytest.raises(HTTPException):
+ await ingest_langflow_chunks(
+ body,
+ authorization=f"Bearer {token}",
+ x_openrag_ingest_token=None,
+ token_service=token_service,
+ writer=writer,
+ )
+
+
+async def test_langflow_ingest_callback_rewrites_langflow_chunk_ids():
+ token_service = LangflowIngestTokenService(secret="test-secret" * 4, ttl_seconds=60)
+ context = DocumentIndexContext(
+ document_id="doc-1",
+ filename="source.pdf",
+ mimetype="application/pdf",
+ embedding_model="text-embedding-3-small",
+ ingest_run_id="run-1",
+ )
+ token = token_service.create_token(context)
+
+ class Writer:
+ def __init__(self):
+ self.calls = []
+
+ async def index_chunks(self, context, chunks, *, final=False):
+ self.calls.append((context, chunks, final))
+ return {"indexed_chunks": len(chunks)}
+
+ writer = Writer()
+ body = LangflowIngestBatch(
+ ingest_run_id="run-1",
+ batch_id=1,
+ final=True,
+ chunks=[
+ LangflowIngestChunk(
+ id="other-doc_0",
+ text="hello",
+ vector=[0.1, 0.2],
+ )
+ ],
+ )
+
+ await ingest_langflow_chunks(
+ body,
+ authorization=f"Bearer {token}",
+ x_openrag_ingest_token=None,
+ token_service=token_service,
+ writer=writer,
+ )
+
+ _, chunks, _ = writer.calls[0]
+ assert chunks[0].chunk_id == "doc-1_1_0"
+ assert chunks[0].metadata["langflow_chunk_id"] == "other-doc_0"
+
+
+async def test_langflow_file_service_sends_backend_callback_global_vars(monkeypatch):
+ token_service = LangflowIngestTokenService(secret="test-secret" * 4, ttl_seconds=60)
+ captured = {}
+
+ class Response:
+ status_code = 200
+ reason_phrase = "OK"
+ headers = {"content-type": "application/json"}
+ text = '{"status":"ok"}'
+
+ def json(self):
+ return {"status": "ok"}
+
+ async def langflow_request(method, endpoint, **kwargs):
+ captured.update({"method": method, "endpoint": endpoint, **kwargs})
+ return Response()
+
+ async def add_provider_credentials_to_headers(*args, **kwargs):
+ return None
+
+ monkeypatch.setattr(
+ "services.langflow_file_service.clients",
+ SimpleNamespace(langflow_request=langflow_request),
+ )
+ monkeypatch.setattr(
+ "utils.langflow_headers.add_provider_credentials_to_headers",
+ add_provider_credentials_to_headers,
+ )
+ monkeypatch.setattr(
+ "config.settings.get_openrag_config",
+ lambda: SimpleNamespace(
+ knowledge=SimpleNamespace(embedding_model="text-embedding-3-small")
+ ),
+ )
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "unit-documents")
+
+ service = LangflowFileService(ingest_token_service=token_service)
+ result = await service.run_ingestion_flow(
+ file_paths=["/tmp/source.pdf"],
+ file_tuples=[("source.pdf", b"content", "application/pdf")],
+ jwt_token="user-token",
+ owner="user-1",
+ owner_name="User One",
+ owner_email="[email protected]",
+ connector_type="local",
+ )
+
+ assert result == {"status": "ok"}
+ payload = captured["json"]
+ assert LangflowFileService.INGEST_OPENSEARCH_COMPONENT_ID not in payload["tweaks"]
+ headers = captured["headers"]
+ assert headers["X-Langflow-Global-Var-OPENRAG_INGEST_URL"].endswith("/internal/ingest/chunks")
+ assert headers["X-Langflow-Global-Var-OPENRAG_INGEST_TOKEN"]
+ assert headers["X-Langflow-Global-Var-OPENRAG_INGEST_RUN_ID"]
+ assert headers["X-Langflow-Global-Var-OPENRAG_INGEST_BATCH_SIZE"]
+
+ decoded_context, _ = token_service.validate_token(
+ headers["X-Langflow-Global-Var-OPENRAG_INGEST_TOKEN"]
+ )
+ assert decoded_context.ingest_run_id == headers["X-Langflow-Global-Var-OPENRAG_INGEST_RUN_ID"]
+ assert decoded_context.owner == "user-1"
+ assert decoded_context.filename == "source.pdf"
+ assert decoded_context.mimetype == "application/pdf"
+ assert decoded_context.file_size == len(b"content")
+ assert decoded_context.index_name == "unit-documents"
+ assert decoded_context.is_sample_data is False
+ assert headers["X-Langflow-Global-Var-DOCUMENT_ID"] == decoded_context.document_id
+
+
+async def test_langflow_file_service_marks_openrag_docs_callback_as_sample_data(monkeypatch):
+ token_service = LangflowIngestTokenService(secret="test-secret" * 4, ttl_seconds=60)
+ captured = {}
+
+ class Response:
+ status_code = 200
+ reason_phrase = "OK"
+ headers = {"content-type": "application/json"}
+ text = '{"status":"ok"}'
+
+ def json(self):
+ return {"status": "ok"}
+
+ async def langflow_request(method, endpoint, **kwargs):
+ captured.update({"method": method, "endpoint": endpoint, **kwargs})
+ return Response()
+
+ async def add_provider_credentials_to_headers(*args, **kwargs):
+ return None
+
+ monkeypatch.setattr(
+ "services.langflow_file_service.clients",
+ SimpleNamespace(langflow_request=langflow_request),
+ )
+ monkeypatch.setattr(
+ "utils.langflow_headers.add_provider_credentials_to_headers",
+ add_provider_credentials_to_headers,
+ )
+ monkeypatch.setattr(
+ "config.settings.get_openrag_config",
+ lambda: SimpleNamespace(
+ knowledge=SimpleNamespace(embedding_model="text-embedding-3-small")
+ ),
+ )
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "unit-documents")
+
+ service = LangflowFileService(ingest_token_service=token_service)
+ await service.run_ingestion_flow(
+ file_paths=["/tmp/source.pdf"],
+ file_tuples=[("source.pdf", b"content", "application/pdf")],
+ jwt_token="user-token",
+ connector_type="openrag_docs",
+ )
+
+ decoded_context, _ = token_service.validate_token(
+ captured["headers"]["X-Langflow-Global-Var-OPENRAG_INGEST_TOKEN"]
+ )
+ assert decoded_context.index_name == "unit-documents"
+ assert decoded_context.is_sample_data is True
+
+
+ ("flow_path", "component_id"),
+ [
+ ("flows/ingestion_flow.json", LangflowFileService.INGEST_OPENSEARCH_COMPONENT_ID),
+ ("flows/openrag_url_mcp.json", LangflowFileService.URL_INGEST_OPENSEARCH_COMPONENT_ID),
+ ],
+)
+def test_ingest_flows_resolve_callback_config_from_global_vars(flow_path, component_id):
+ flow = json.loads(Path(flow_path).read_text(encoding="utf-8"))
+ node = next(
+ node
+ for node in flow["data"]["nodes"]
+ if node.get("id") == component_id
+ and node.get("data", {}).get("node", {}).get("display_name")
+ == "OpenSearch (Multi-Model Multi-Embedding)"
+ )
+ template = node["data"]["node"]["template"]
+
+ assert template["openrag_ingest_url"]["value"] == "OPENRAG_INGEST_URL"
+ assert template["openrag_ingest_token"]["value"] == "OPENRAG_INGEST_TOKEN"
+ assert template["openrag_ingest_run_id"]["value"] == "OPENRAG_INGEST_RUN_ID"
+ assert template["openrag_ingest_url"]["load_from_db"] is True
+ assert template["openrag_ingest_token"]["load_from_db"] is True
+ assert template["openrag_ingest_run_id"]["load_from_db"] is True
+ assert template["openrag_ingest_url"]["input_types"] == ["Text", "Message"]
+ assert template["openrag_ingest_token"]["input_types"] == ["Text", "Message"]
+ assert template["openrag_ingest_run_id"]["input_types"] == ["Text", "Message"]
+ assert template["openrag_ingest_token"]["_input_type"] == "StrInput"
+ assert "OPENRAG_INGEST_URL" in template["code"]["value"]
+ assert "_openrag_ingest_global_placeholders" in template["code"]["value"]
+ assert 'url = self._openrag_callback_value("openrag_ingest_url")' in template["code"]["value"]
+ assert (
+ 'token = self._openrag_callback_value("openrag_ingest_token")' in template["code"]["value"]
+ )
+ assert (
+ 'ingest_run_id = self._openrag_callback_value("openrag_ingest_run_id")'
+ in template["code"]["value"]
+ )
+ assert 'url = (self.openrag_ingest_url or "").strip()' not in template["code"]["value"]
+ assert 'token = (self.openrag_ingest_token or "").strip()' not in template["code"]["value"]
+ assert (
+ 'ingest_run_id = (self.openrag_ingest_run_id or "").strip()'
+ not in template["code"]["value"]
+ )
+ assert "value.lower() in" not in template["code"]["value"]
+
+
+ ("flow_path", "component_id"),
+ [
+ ("flows/ingestion_flow.json", LangflowFileService.INGEST_OPENSEARCH_COMPONENT_ID),
+ ("flows/openrag_url_mcp.json", LangflowFileService.URL_INGEST_OPENSEARCH_COMPONENT_ID),
+ ],
+)
+def test_ingest_flows_wire_callback_global_vars_into_opensearch(flow_path, component_id):
+ flow = json.loads(Path(flow_path).read_text(encoding="utf-8"))
+ nodes = {node.get("id"): node for node in flow["data"]["nodes"]}
+ edges = flow["data"]["edges"]
+ expected = {
+ "openrag_ingest_url": "OPENRAG_INGEST_URL",
+ "openrag_ingest_token": "OPENRAG_INGEST_TOKEN",
+ "openrag_ingest_run_id": "OPENRAG_INGEST_RUN_ID",
+ }
+
+ for field_name, variable_name in expected.items():
+ source_id = f"TextInput-OpenRAGIngest-{field_name.removeprefix('openrag_ingest_')}"
+ text_node = nodes[source_id]
+ input_template = text_node["data"]["node"]["template"]["input_value"]
+ assert input_template["value"] == variable_name
+ assert input_template["load_from_db"] is True
+
+ assert any(
+ edge.get("source") == source_id
+ and edge.get("target") == component_id
+ and edge.get("data", {}).get("targetHandle", {}).get("fieldName") == field_name
+ for edge in edges
+ )
+
+
+ "config_path",
+ [
+ "docker-compose.yml",
+ "kubernetes/helm/openrag/values.yaml",
+ "kubernetes/operator/internal/controller/env.go",
+ ],
+)
+def test_langflow_callback_global_vars_are_allowlisted(config_path):
+ config_text = Path(config_path).read_text(encoding="utf-8")
+
+ assert (
+ "LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT" in config_text
+ or "variablesToGetFromEnvironment" in config_text
+ )
+ for variable_name in CALLBACK_GLOBAL_VARS:
+ assert variable_name in config_text
+
+
+ "config_path",
+ [
+ "docker-compose.yml",
+ "kubernetes/helm/openrag/templates/langflow/langflow-dotenv.yaml",
+ "kubernetes/operator/internal/controller/env.go",
+ ],
+)
+def test_langflow_callback_global_vars_have_runtime_placeholders(config_path):
+ config_text = Path(config_path).read_text(encoding="utf-8")
+
+ for variable_name in CALLBACK_GLOBAL_VARS:
+ assert f"{variable_name}=" in config_text or f'"{variable_name}":' in config_text
diff --git a/tests/unit/test_oauth_encryption.py b/tests/unit/test_oauth_encryption.py
index 2d0b6c0d0..ba44fe48d 100644
--- a/tests/unit/test_oauth_encryption.py
+++ b/tests/unit/test_oauth_encryption.py
@@ -1,66 +1,101 @@
import json
import pytest
+
from connectors.google_drive.oauth import GoogleDriveOAuth
from connectors.onedrive.oauth import OneDriveOAuth
+
@pytest.mark.asyncio
async def test_google_drive_auth_upgrade(tmp_path, monkeypatch):
"""Test that Google Drive credentials are encrypted on load if not already."""
import utils.encryption
+
# Reset internal cache to ensure it reads from env
utils.encryption._cached_master_secret = None
- monkeypatch.setenv("OPENRAG_ENCRYPTION_KEY", "dGVzdC1rZXktMzItYnl0ZXMtZm9yLXVuaXQtdGVzdGluZy1vbmx5") # 32-byte base64
+ monkeypatch.setenv(
+ "OPENRAG_ENCRYPTION_KEY",
+ "dGVzdC1rZXktMzItYnl0ZXMtZm9yLXVuaXQtdGVzdGluZy1vbmx5",
+ ) # 32-byte base64
print("Testing Google Drive Auth Upgrade...")
g_token_path = tmp_path / "fake_gdrive_token.json"
g_plain = {
"token": "fake-google-token",
"refresh_token": "fake-refresh",
- "scopes": ["some-scope"],
- "expiry": "2028-03-02T15:46:11.368083"
+ "scopes": GoogleDriveOAuth.SCOPES,
+ "expiry": "2028-03-02T15:46:11.368083",
}
with open(g_token_path, "w") as f:
json.dump(g_plain, f)
-
+
g_oauth = GoogleDriveOAuth(client_id="abc", client_secret="def", token_file=str(g_token_path))
- try:
- await g_oauth.load_credentials()
- except ValueError:
- print("Expected network validation error caught. Proceeding to verify disk encryption...")
-
+ await g_oauth.load_credentials()
+
# Check if upgraded
- with open(g_token_path, "r") as f:
+ with open(g_token_path) as f:
upgraded = json.load(f)
assert "algorithm" in upgraded and upgraded["algorithm"] == "AES-256-GCM"
print("Google Drive Auth OK")
+
+async def test_google_drive_auth_missing_scopes_forces_reauth(tmp_path, monkeypatch):
+ import utils.encryption
+
+ utils.encryption._cached_master_secret = None
+ monkeypatch.setenv(
+ "OPENRAG_ENCRYPTION_KEY",
+ "dGVzdC1rZXktMzItYnl0ZXMtZm9yLXVuaXQtdGVzdGluZy1vbmx5",
+ )
+
+ g_token_path = tmp_path / "fake_gdrive_token.json"
+ g_plain = {
+ "token": "old-google-token",
+ "refresh_token": "old-refresh",
+ "scopes": ["https://www.googleapis.com/auth/drive.readonly"],
+ "expiry": "2028-03-02T15:46:11.368083",
+ }
+ with open(g_token_path, "w") as f:
+ json.dump(g_plain, f)
+
+ g_oauth = GoogleDriveOAuth(client_id="abc", client_secret="def", token_file=str(g_token_path))
+
+ assert await g_oauth.load_credentials() is None
+ assert g_oauth.creds is None
+ assert not g_token_path.exists()
+
+
@pytest.mark.asyncio
async def test_msal_auth_upgrade(tmp_path, monkeypatch):
"""Test that MSAL credentials are encrypted on load if not already."""
import utils.encryption
+
# Reset internal cache to ensure it reads from env
utils.encryption._cached_master_secret = None
- monkeypatch.setenv("OPENRAG_ENCRYPTION_KEY", "dGVzdC1rZXktMzItYnl0ZXMtZm9yLXVuaXQtdGVzdGluZy1vbmx5") # 32-byte base64
+ monkeypatch.setenv(
+ "OPENRAG_ENCRYPTION_KEY",
+ "dGVzdC1rZXktMzItYnl0ZXMtZm9yLXVuaXQtdGVzdGluZy1vbmx5",
+ ) # 32-byte base64
print("Testing MSAL Auth Upgrade...")
m_token_path = tmp_path / "fake_msal_token.json"
- m_plain = {
- "refresh_token": "legacy-flat-refresh-token"
- }
+ m_plain = {"refresh_token": "legacy-flat-refresh-token"}
with open(m_token_path, "w") as f:
json.dump(m_plain, f)
-
+
m_oauth = OneDriveOAuth(client_id="abc", client_secret="def", token_file=str(m_token_path))
-
+
# We monkey patch MSAL acquire_token_by_refresh_token to pretend it worked
- m_oauth.app.acquire_token_by_refresh_token = lambda refresh_token, scopes: {"access_token": "new-access-token"}
+ m_oauth.app.acquire_token_by_refresh_token = lambda refresh_token, scopes: {
+ "access_token": "new-access-token"
+ }
m_oauth.app.get_accounts = lambda: [{"username": "fake-user"}]
-
+
result = await m_oauth.load_credentials()
assert result
-
- with open(m_token_path, "r") as f:
+
+ with open(m_token_path) as f:
upgraded_m = json.load(f)
assert "algorithm" in upgraded_m and upgraded_m["algorithm"] == "AES-256-GCM"
print("MSAL Auth OK")
diff --git a/tests/unit/test_onedrive_connector.py b/tests/unit/test_onedrive_connector.py
new file mode 100644
index 000000000..d1838459e
--- /dev/null
+++ b/tests/unit/test_onedrive_connector.py
@@ -0,0 +1,118 @@
+from unittest.mock import AsyncMock
+
+import pytest
+
+
+class _GraphResponse:
+ def __init__(self, status_code: int, payload: dict | None = None):
+ self.status_code = status_code
+ self._payload = payload or {}
+ self.text = str(self._payload)
+
+ def json(self) -> dict:
+ return self._payload
+
+
+async def test_onedrive_cached_download_path_returns_document_with_empty_acl(tmp_path):
+ from connectors.onedrive.connector import OneDriveConnector
+
+ connector = OneDriveConnector({"token_file": str(tmp_path / "token.json")})
+ connector.authenticate = AsyncMock(return_value=True)
+ connector._download_file_from_url = AsyncMock(return_value=b"cached bytes")
+ connector.set_file_infos(
+ [
+ {
+ "id": "cached-file",
+ "name": "cached.pdf",
+ "mimeType": "application/pdf",
+ "downloadUrl": "https://download.example/cached.pdf",
+ "webUrl": "https://onedrive.example/cached.pdf",
+ "size": 12,
+ }
+ ]
+ )
+
+ doc = await connector.get_file_content("cached-file")
+
+ connector._download_file_from_url.assert_awaited_once_with(
+ "https://download.example/cached.pdf"
+ )
+ assert doc.id == "cached-file"
+ assert doc.filename == "cached.pdf"
+ assert doc.content == b"cached bytes"
+ assert doc.acl.owner == ""
+ assert doc.acl.allowed_users == []
+ assert doc.acl.allowed_groups == []
+ assert doc.acl.allowed_principals == []
+
+
+async def test_onedrive_fetch_item_metadata_uses_item_id_for_drive_scoped_ids(tmp_path):
+ from connectors.onedrive.connector import OneDriveConnector
+
+ connector = OneDriveConnector({"token_file": str(tmp_path / "token.json")})
+ connector._make_graph_request = AsyncMock(
+ return_value=_GraphResponse(200, {"id": "item-id", "file": {}})
+ )
+
+ item = await connector._fetch_item_metadata("drive-id!item-id")
+
+ assert item == {"id": "item-id", "file": {}}
+ connector._make_graph_request.assert_awaited_once()
+ url = connector._make_graph_request.await_args.args[0]
+ assert url == "https://graph.microsoft.com/v1.0/drives/drive-id/items/item-id"
+
+
+async def test_onedrive_fetch_item_metadata_strips_sharing_prefix_only_after_full_item_id_fails(
+ tmp_path,
+):
+ from connectors.onedrive.connector import OneDriveConnector
+
+ connector = OneDriveConnector({"token_file": str(tmp_path / "token.json")})
+ connector._make_graph_request = AsyncMock(
+ side_effect=[
+ _GraphResponse(404),
+ _GraphResponse(404),
+ _GraphResponse(404),
+ _GraphResponse(404),
+ _GraphResponse(404),
+ _GraphResponse(200, {"id": "item-id", "file": {}}),
+ ]
+ )
+
+ item = await connector._fetch_item_metadata("drive-id!sitem-id")
+
+ assert item == {"id": "item-id", "file": {}}
+ urls = [call.args[0] for call in connector._make_graph_request.await_args_list]
+ assert urls[4] == "https://graph.microsoft.com/v1.0/drives/drive-id/items/sitem-id"
+ assert urls[5] == "https://graph.microsoft.com/v1.0/drives/drive-id/items/item-id"
+
+
+async def test_onedrive_sharing_id_fallback_returns_document_with_empty_acl(tmp_path):
+ from connectors.onedrive.connector import OneDriveConnector
+
+ class OAuth:
+ def get_access_token(self):
+ return "access-token"
+
+ connector = OneDriveConnector({"token_file": str(tmp_path / "token.json")})
+ connector.oauth = OAuth()
+ connector.authenticate = AsyncMock(return_value=True)
+ connector._get_file_metadata_by_id = AsyncMock(return_value=None)
+ connector._download_via_shares_endpoint = AsyncMock(return_value=b"shared bytes")
+
+ doc = await connector.get_file_content("drive-id!shared-item")
+
+ connector._download_via_shares_endpoint.assert_awaited_once_with(
+ "drive-id!shared-item",
+ {"Authorization": "Bearer access-token"},
+ )
+ assert doc.id == "drive-id!shared-item"
+ assert doc.content == b"shared bytes"
+ assert doc.acl.owner == ""
+ assert doc.acl.allowed_users == []
+ assert doc.acl.allowed_groups == []
+ assert doc.acl.allowed_principals == []
diff --git a/tests/unit/test_processor_mapping_client.py b/tests/unit/test_processor_mapping_client.py
new file mode 100644
index 000000000..24e45c38f
--- /dev/null
+++ b/tests/unit/test_processor_mapping_client.py
@@ -0,0 +1,138 @@
+from types import SimpleNamespace
+
+import pytest
+
+from models.processors import TaskProcessor
+from services.document_index_writer import DocumentIndexWriter
+
+
+async def test_standard_processor_uses_shared_writer_for_embedding_mapping_and_writes(
+ tmp_path,
+ monkeypatch,
+):
+ user_client = SimpleNamespace(
+ search_calls=[],
+ index_calls=[],
+ )
+ admin_client = SimpleNamespace(bulk_calls=[], refresh_calls=[])
+ mapping_clients = []
+
+ async def search(**kwargs):
+ user_client.search_calls.append(kwargs)
+ return {"_scroll_id": None, "hits": {"hits": []}}
+
+ async def index(**kwargs):
+ user_client.index_calls.append(kwargs)
+
+ user_client.search = search
+ user_client.index = index
+
+ class Indices:
+ async def exists(self, *, index):
+ return True
+
+ async def refresh(self, *, index):
+ admin_client.refresh_calls.append({"index": index})
+
+ async def bulk(**kwargs):
+ admin_client.bulk_calls.append(kwargs)
+ return {"errors": False, "items": []}
+
+ admin_client.indices = Indices()
+ admin_client.bulk = bulk
+
+ class SessionManager:
+ def get_user_opensearch_client(self, user_id, jwt_token):
+ assert user_id == "user-1"
+ assert jwt_token == "Bearer user-token"
+ return user_client
+
+ class ModelsService:
+ async def get_litellm_model_name(self, embedding_model):
+ return embedding_model
+
+ class EmbeddingClient:
+ class Embeddings:
+ async def create(self, model, input):
+ return SimpleNamespace(
+ data=[SimpleNamespace(embedding=[0.1, 0.2, 0.3]) for _ in input]
+ )
+
+ embeddings = Embeddings()
+
+ async def ensure_embedding_field_exists(client, model_name, index_name, dimensions):
+ mapping_clients.append(client)
+ assert model_name == "text-embedding-3-small"
+ assert index_name == "documents"
+ assert dimensions == 3
+ return "chunk_embedding_text_embedding_3_small"
+
+ monkeypatch.setattr(
+ "config.settings.clients",
+ SimpleNamespace(
+ opensearch=admin_client,
+ patched_embedding_client=EmbeddingClient(),
+ ),
+ )
+ monkeypatch.setattr(
+ "models.processors.clients",
+ SimpleNamespace(
+ opensearch=admin_client,
+ patched_embedding_client=EmbeddingClient(),
+ ),
+ )
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents")
+ monkeypatch.setattr("models.processors.get_index_name", lambda: "documents")
+ monkeypatch.setattr(
+ "services.document_service.get_embedding_model",
+ lambda: "text-embedding-3-small",
+ )
+ monkeypatch.setattr(
+ "models.processors.get_openrag_config",
+ lambda: SimpleNamespace(knowledge=SimpleNamespace(embedding_model="")),
+ )
+ monkeypatch.setattr(
+ "services.document_index_writer.ensure_embedding_field_exists",
+ ensure_embedding_field_exists,
+ )
+
+ file_path = tmp_path / "doc.md"
+ file_path.write_text("# Test\n\nhello world", encoding="utf-8")
+ document_service = SimpleNamespace(
+ session_manager=SessionManager(),
+ document_index_writer=DocumentIndexWriter(opensearch_client=admin_client),
+ )
+ processor = TaskProcessor(
+ document_service=document_service,
+ models_service=ModelsService(),
+ docling_service=None,
+ )
+
+ result = await processor.process_document_standard(
+ file_path=str(file_path),
+ file_hash="file-1",
+ owner_user_id="user-1",
+ original_filename="doc.md",
+ jwt_token="Bearer user-token",
+ embedding_model="text-embedding-3-small",
+ )
+
+ assert result == {"status": "indexed", "id": "file-1"}
+ assert mapping_clients == [admin_client]
+ assert user_client.search_calls[0] == {
+ "index": "documents",
+ "body": {
+ "size": 1,
+ "_source": False,
+ "query": {"term": {"document_id": "file-1"}},
+ },
+ }
+ assert user_client.search_calls[1]["body"]["query"] == {"term": {"document_id": "file-1"}}
+ assert user_client.index_calls == []
+ assert admin_client.bulk_calls
+ bulk_body = admin_client.bulk_calls[0]["body"]
+ assert bulk_body[0] == {"index": {"_index": "documents", "_id": "file-1_0"}}
+ assert bulk_body[1]["document_id"] == "file-1"
+ assert bulk_body[1]["owner"] == "user-1"
+ assert bulk_body[1]["chunk_embedding_text_embedding_3_small"] == [0.1, 0.2, 0.3]
diff --git a/tests/unit/test_processors_clear_stale_chunks.py b/tests/unit/test_processors_clear_stale_chunks.py
index f1daac461..b25efe16d 100644
--- a/tests/unit/test_processors_clear_stale_chunks.py
+++ b/tests/unit/test_processors_clear_stale_chunks.py
@@ -1,11 +1,11 @@
"""Re-indexing in `process_document_standard` must clear prior chunks for the
same `document_id` before writing the new ones.
-Per-chunk indexing uses ids `{file_hash}_{i}` with `opensearch_client.index`
-(upsert). Without a pre-delete, a re-index that produces fewer chunks than the
-prior pass leaves trailing chunks `{file_hash}_{N..M-1}` behind with the OLD
-metadata — most visibly, the old filename after a SharePoint rename. This test
-pins the invariant introduced to fix that.
+Chunk indexing uses ids `{file_hash}_{i}` through the centralized
+DocumentIndexWriter upsert path. Without a pre-delete, a re-index that produces
+fewer chunks than the prior pass leaves trailing chunks `{file_hash}_{N..M-1}`
+behind with the OLD metadata — most visibly, the old filename after a
+SharePoint rename. This test pins that invariant.
Pins: `src/models/processors.py` :: TaskProcessor.process_document_standard.
"""
@@ -30,7 +30,7 @@ def _make_processor_with_mocks():
opensearch_client = AsyncMock()
# exists() is checked at the top of process_document_standard; returning
- # False forces the re-index path (the path where delete_by_query must fire).
+ # False forces the re-index path (the path where stale chunks are cleared).
opensearch_client.exists = AsyncMock(return_value=False)
session_manager = MagicMock()
@@ -38,6 +38,7 @@ def _make_processor_with_mocks():
document_service = MagicMock()
document_service.session_manager = session_manager
+ document_service.document_index_writer = None
models_service = MagicMock()
models_service.get_litellm_model_name = AsyncMock(return_value="text-embedding-3-small")
@@ -52,7 +53,7 @@ def _make_processor_with_mocks():
return processor, opensearch_client
-def _patch_embedding_pipeline(monkeypatch, chunk_count: int):
+def _patch_embedding_pipeline(monkeypatch, chunk_count: int, write_client=None):
"""Stub out the docling / embedding / index-mapping side of
process_document_standard so the test focuses on the OpenSearch write
ordering. `chunk_count` controls how many chunks the simulated text-file
@@ -75,14 +76,6 @@ def _patch_embedding_pipeline(monkeypatch, chunk_count: int):
monkeypatch.setattr(processors_mod, "get_embedding_model", lambda: "text-embedding-3-small")
monkeypatch.setattr(processors_mod, "get_index_name", lambda: "test-index")
- # Field-mapping helper writes nothing useful for this test.
- async def _ensure_embedding_field_exists(_client, _model, _index, _dims):
- return "embedding_field"
-
- monkeypatch.setattr(
- processors_mod, "ensure_embedding_field_exists", _ensure_embedding_field_exists
- )
-
# chunk_texts_for_embeddings is imported lazily inside the function from
# services.document_service — patch it at its source.
from services import document_service as ds_mod
@@ -106,21 +99,21 @@ def _patch_embedding_pipeline(monkeypatch, chunk_count: int):
)
fake_clients = MagicMock()
fake_clients.patched_embedding_client = fake_embed_client
+ fake_clients.opensearch = write_client
monkeypatch.setattr(processors_mod, "clients", fake_clients)
@pytest.mark.asyncio
async def test_stale_chunks_cleared_before_reindex(monkeypatch):
- """Stale chunks must be cleared (via primary-id deletes) before any new
- `index()` call so prior chunks (e.g. with an old filename after a rename)
- cannot survive the per-chunk upsert.
+ """Stale chunks must be cleared (via primary-id deletes) before the writer
+ upsert so prior chunks cannot survive a re-index with fewer chunks.
DLS-safe pattern: enumerate visible chunk _ids via search, then issue a
`delete` per primary `_id`. `delete_by_query` is silently filtered under
DLS and must NOT be used.
"""
processor, opensearch_client = _make_processor_with_mocks()
- _patch_embedding_pipeline(monkeypatch, chunk_count=3)
+ _patch_embedding_pipeline(monkeypatch, chunk_count=3, write_client=opensearch_client)
stale_chunk_ids = ["abc123_0", "abc123_1", "abc123_2", "abc123_3", "abc123_4"]
op_order: list[tuple[str, dict]] = []
@@ -135,12 +128,23 @@ async def test_stale_chunks_cleared_before_reindex(monkeypatch):
op_order.append(("delete", kw))
return {"result": "deleted"}
- async def _index(**kw):
- op_order.append(("index", kw))
+ class _FakeDocumentIndexWriter:
+ async def index_chunks(self, context, chunks, *, final=False):
+ op_order.append(
+ (
+ "index",
+ {
+ "context": context,
+ "chunks": chunks,
+ "final": final,
+ },
+ )
+ )
+ return {"indexed_chunks": len(chunks)}
opensearch_client.search = AsyncMock(side_effect=_search)
opensearch_client.delete = AsyncMock(side_effect=_delete)
- opensearch_client.index = AsyncMock(side_effect=_index)
+ processor.document_service.document_index_writer = _FakeDocumentIndexWriter()
with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:
tmp.write(b"hello world")
@@ -183,7 +187,14 @@ async def test_stale_chunks_cleared_before_reindex(monkeypatch):
assert call["id"] == expected_id
assert call.get("refresh") is True
- # 4) delete_by_query must NEVER be used (DLS would silently filter it).
+ # 4) The centralized writer receives the new chunks after cleanup.
+ index_call = next(kw for op, kw in op_order if op == "index")
+ assert index_call["context"].document_id == "abc123"
+ assert index_call["context"].filename == "renamed.txt"
+ assert len(index_call["chunks"]) == 3
+ assert index_call["final"] is True
+
+ # 5) delete_by_query must NEVER be used (DLS would silently filter it).
if hasattr(opensearch_client, "delete_by_query"):
opensearch_client.delete_by_query.assert_not_called()
@@ -193,14 +204,20 @@ async def test_delete_failure_does_not_abort_reindex(monkeypatch):
"""A transient delete failure must be logged and swallowed — the per-chunk
upsert still runs so the sync isn't worse off than today's behavior."""
processor, opensearch_client = _make_processor_with_mocks()
- _patch_embedding_pipeline(monkeypatch, chunk_count=2)
+ _patch_embedding_pipeline(monkeypatch, chunk_count=2, write_client=opensearch_client)
# Have the enumerate step itself blow up — that's the only "delete failure"
# surface the helper exposes, since per-id delete swallows NotFoundError.
opensearch_client.search = AsyncMock(side_effect=RuntimeError("os 503"))
opensearch_client.delete = AsyncMock()
index_calls: list[dict] = []
- opensearch_client.index = AsyncMock(side_effect=lambda **kw: index_calls.append(kw))
+
+ class _FakeDocumentIndexWriter:
+ async def index_chunks(self, context, chunks, *, final=False):
+ index_calls.append({"context": context, "chunks": chunks, "final": final})
+ return {"indexed_chunks": len(chunks)}
+
+ processor.document_service.document_index_writer = _FakeDocumentIndexWriter()
with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:
tmp.write(b"hello world")
@@ -218,23 +235,28 @@ async def test_delete_failure_does_not_abort_reindex(monkeypatch):
Path(tmp_path).unlink(missing_ok=True)
assert result["status"] == "indexed"
- assert len(index_calls) == 2, "indexing must still happen if the pre-delete fails"
+ assert len(index_calls) == 1, "indexing must still happen if the pre-delete fails"
+ assert len(index_calls[0]["chunks"]) == 2
+ assert index_calls[0]["final"] is True
@pytest.mark.asyncio
async def test_connector_file_id_stored_in_chunk_when_provided(monkeypatch):
- """When connector_file_id is passed to process_document_standard, every
- indexed chunk must carry that value so the 404 cleanup path can find and
- delete chunks by their connector source ID."""
+ """Connector reindex uses the file hash as document_id and stores the
+ upstream connector ID separately so orphan cleanup can query connector_file_id."""
processor, opensearch_client = _make_processor_with_mocks()
- _patch_embedding_pipeline(monkeypatch, chunk_count=2)
+ _patch_embedding_pipeline(monkeypatch, chunk_count=2, write_client=opensearch_client)
opensearch_client.search = AsyncMock(return_value={"_scroll_id": None, "hits": {"hits": []}})
opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})
- indexed_bodies: list[dict] = []
- opensearch_client.index = AsyncMock(
- side_effect=lambda **kw: indexed_bodies.append(kw.get("body", {}))
- )
+ index_calls: list[dict] = []
+
+ class _FakeDocumentIndexWriter:
+ async def index_chunks(self, context, chunks, *, final=False):
+ index_calls.append({"context": context, "chunks": chunks, "final": final})
+ return {"indexed_chunks": len(chunks)}
+
+ processor.document_service.document_index_writer = _FakeDocumentIndexWriter()
with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:
tmp.write(b"hello world")
@@ -253,27 +275,30 @@ async def test_connector_file_id_stored_in_chunk_when_provided(monkeypatch):
Path(tmp_path).unlink(missing_ok=True)
assert result["status"] == "indexed"
- assert len(indexed_bodies) == 2, "expected one index call per chunk"
- for body in indexed_bodies:
- assert body.get("connector_file_id") == "sharepoint-item-xyz", (
- f"connector_file_id missing or wrong in chunk: {body}"
- )
- assert body.get("document_id") == "sha-abc", "document_id must still be the content hash"
+ assert len(index_calls) == 1
+ assert index_calls[0]["context"].document_id == "sha-abc"
+ assert len(index_calls[0]["chunks"]) == 2
+ for chunk in index_calls[0]["chunks"]:
+ assert chunk.metadata["connector_file_id"] == "sharepoint-item-xyz"
@pytest.mark.asyncio
async def test_connector_file_id_absent_when_not_provided(monkeypatch):
- """When connector_file_id is not passed (local uploads, non-connector paths),
- the field must be absent from indexed chunks — no None/empty pollution."""
+ """Local uploads and other non-connector paths should not write an empty
+ connector_file_id marker."""
processor, opensearch_client = _make_processor_with_mocks()
- _patch_embedding_pipeline(monkeypatch, chunk_count=1)
+ _patch_embedding_pipeline(monkeypatch, chunk_count=1, write_client=opensearch_client)
opensearch_client.search = AsyncMock(return_value={"_scroll_id": None, "hits": {"hits": []}})
opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})
- indexed_bodies: list[dict] = []
- opensearch_client.index = AsyncMock(
- side_effect=lambda **kw: indexed_bodies.append(kw.get("body", {}))
- )
+ index_calls: list[dict] = []
+
+ class _FakeDocumentIndexWriter:
+ async def index_chunks(self, context, chunks, *, final=False):
+ index_calls.append({"context": context, "chunks": chunks, "final": final})
+ return {"indexed_chunks": len(chunks)}
+
+ processor.document_service.document_index_writer = _FakeDocumentIndexWriter()
with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:
tmp.write(b"hello world")
@@ -288,5 +313,36 @@ async def test_connector_file_id_absent_when_not_provided(monkeypatch):
finally:
Path(tmp_path).unlink(missing_ok=True)
- assert len(indexed_bodies) == 1
- assert "connector_file_id" not in indexed_bodies[0]
+ assert len(index_calls) == 1
+ assert index_calls[0]["chunks"][0].metadata == {}
+
+
+def test_document_index_writer_outputs_connector_file_id_when_present():
+ from services.document_index_writer import (
+ DocumentIndexChunk,
+ DocumentIndexContext,
+ DocumentIndexWriter,
+ )
+
+ writer = DocumentIndexWriter()
+ doc = writer._build_chunk_document(
+ context=DocumentIndexContext(
+ document_id="sha-abc",
+ filename="report.txt",
+ mimetype="text/plain",
+ embedding_model="text-embedding-3-small",
+ owner="alice",
+ ),
+ chunk=DocumentIndexChunk(
+ chunk_id="sha-abc_0",
+ text="hello",
+ vector=[0.1, 0.2, 0.3],
+ page=1,
+ metadata={"connector_file_id": "sharepoint-item-xyz"},
+ ),
+ embedding_field="chunk_embedding_text_embedding_3_small",
+ indexed_time="2026-05-28T00:00:00+00:00",
+ )
+
+ assert doc["document_id"] == "sha-abc"
+ assert doc["connector_file_id"] == "sharepoint-item-xyz"
diff --git a/tests/unit/test_skip_os_security_setup_init_index.py b/tests/unit/test_skip_os_security_setup_init_index.py
index 531a4fc8a..55381b35b 100644
--- a/tests/unit/test_skip_os_security_setup_init_index.py
+++ b/tests/unit/test_skip_os_security_setup_init_index.py
@@ -11,6 +11,7 @@ Three cases:
import sys
from pathlib import Path
+from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -32,6 +33,16 @@ def _fake_os_client():
return client
+def _patch_config(monkeypatch, init_mod):
+ monkeypatch.setattr(
+ init_mod,
+ "get_openrag_config",
+ lambda: SimpleNamespace(
+ knowledge=SimpleNamespace(embedding_model="text-embedding-3-small")
+ ),
+ )
+
+
@pytest.mark.asyncio
async def test_security_setup_called_when_flag_false(monkeypatch):
import utils.opensearch_init as init_mod
@@ -39,6 +50,7 @@ async def test_security_setup_called_when_flag_false(monkeypatch):
monkeypatch.setattr(init_mod, "OPENRAG_SKIP_OS_SECURITY_SETUP", False)
monkeypatch.setattr(init_mod, "IBM_AUTH_ENABLED", False)
monkeypatch.setattr(init_mod, "PLATFORM_AUTH_DEV_MODE", False)
+ _patch_config(monkeypatch, init_mod)
os_client = _fake_os_client()
setup_mock = AsyncMock()
@@ -66,6 +78,7 @@ async def test_security_setup_skipped_when_flag_true(monkeypatch):
monkeypatch.setattr(init_mod, "OPENRAG_SKIP_OS_SECURITY_SETUP", True)
monkeypatch.setattr(init_mod, "IBM_AUTH_ENABLED", False)
monkeypatch.setattr(init_mod, "PLATFORM_AUTH_DEV_MODE", False)
+ _patch_config(monkeypatch, init_mod)
os_client = _fake_os_client()
setup_mock = AsyncMock()
@@ -98,6 +111,7 @@ async def test_index_creation_still_runs_when_flag_true(monkeypatch):
monkeypatch.setattr(init_mod, "OPENRAG_SKIP_OS_SECURITY_SETUP", True)
monkeypatch.setattr(init_mod, "IBM_AUTH_ENABLED", False)
monkeypatch.setattr(init_mod, "PLATFORM_AUTH_DEV_MODE", False)
+ _patch_config(monkeypatch, init_mod)
os_client = _fake_os_client()