Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: upgrade opensearch to 3.8.0, fix length over 256 on roles (#2368)

Lucas Oliveira57551f562058
diff --git a/Dockerfile b/Dockerfile
index 9799d4083..02d041abe 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
########################################
# Stage 1: Upstream OpenSearch with plugins
########################################
-FROM opensearchproject/opensearch:3.6.0 AS upstream_opensearch
+FROM opensearchproject/opensearch:3.8.0 AS upstream_opensearch
# Remove plugins
RUN opensearch-plugin remove opensearch-neural-search || true && \
@@ -9,21 +9,21 @@ RUN opensearch-plugin remove opensearch-neural-search || true && \
# Prepare jvector plugin artifacts
RUN mkdir -p /tmp/opensearch-jvector-plugin && \
- curl -L -s https://github.com/opensearch-project/opensearch-jvector/releases/download/3.6.0.0/artifacts.tar.gz \
+ curl -L -s https://github.com/opensearch-project/opensearch-jvector/releases/download/3.8.0.0/artifacts.tar.gz \
| tar zxvf - -C /tmp/opensearch-jvector-plugin
# Prepare neural-search plugin
RUN mkdir -p /tmp/opensearch-neural-search && \
- curl -L -s https://github.com/IBM/neural-search-jvector/releases/download/3.6.0.0/opensearch-neural-search-3.6.0.0.zip \
+ curl -L -s https://github.com/IBM/neural-search-jvector/releases/download/3.8.0.0/opensearch-neural-search-jvector-3.8.0.0.zip \
> /tmp/opensearch-neural-search/plugin.zip
# Install additional plugins
-RUN opensearch-plugin install --batch file:///tmp/opensearch-jvector-plugin/repository/org/opensearch/plugin/opensearch-jvector-plugin/3.6.0.0/opensearch-jvector-plugin-3.6.0.0.zip && \
+RUN opensearch-plugin install --batch file:///tmp/opensearch-jvector-plugin/repository/org/opensearch/plugin/opensearch-jvector-plugin/3.8.0.0/opensearch-jvector-plugin-3.8.0.0.zip && \
opensearch-plugin install --batch file:///tmp/opensearch-neural-search/plugin.zip && \
opensearch-plugin install --batch repository-gcs && \
opensearch-plugin install --batch repository-azure && \
# opensearch-plugin install --batch repository-s3 && \
- opensearch-plugin install --batch https://github.com/opensearch-project/opensearch-prometheus-exporter/releases/download/3.6.0.0/prometheus-exporter-3.6.0.0.zip
+ opensearch-plugin install --batch https://github.com/opensearch-project/opensearch-prometheus-exporter/releases/download/3.8.0.0/prometheus-exporter-3.8.0.0.zip
# Apply Netty patch
COPY patch-netty.sh /tmp/
diff --git a/cloud_securityconfig/roles.yml b/cloud_securityconfig/roles.yml
index 1503ea891..20a65556e 100644
--- a/cloud_securityconfig/roles.yml
+++ b/cloud_securityconfig/roles.yml
@@ -3,7 +3,7 @@ _meta:
config_version: 2
openrag_user_role:
- description: "DLS: user can read docs they own or are allowed on"
+ description: "DLS: user can read docs they own or unowned docs"
cluster_permissions:
- "indices:data/read/scroll"
- "indices:data/read/scroll/clear"
@@ -17,9 +17,6 @@ openrag_user_role:
{"bool":{"should":[
{"term":{"owner":"${user.name}"}},
{"term":{"owner":"${attr.jwt.email}"}},
- {"term":{"allowed_users":"${user.name}"}},
- {"term":{"allowed_users":"${attr.jwt.email}"}},
- {"terms":{"allowed_principals":{"index":"openrag_dls_principals","id":"${user.name}","path":"principals"}}},
{"bool":{"must_not":{"exists":{"field":"owner"}}}}
],"minimum_should_match":1}}
- index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*", "orag-*", "orag*"]
@@ -50,3 +47,18 @@ openrag_user_role:
- indices:admin/mappings/get
- indices:admin/exists
- indices:admin/get
+
+openrag_user_acl_role:
+ description: "DLS: user can read docs they are allowed on via ACL"
+ cluster_permissions: []
+
+ index_permissions:
+ - index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*", "orag-*", "orag*"]
+ allowed_actions:
+ - read
+ dls: >
+ {"bool":{"should":[
+ {"term":{"allowed_users":"${user.name}"}},
+ {"term":{"allowed_users":"${attr.jwt.email}"}},
+ {"terms":{"allowed_principals":{"index":"openrag_dls_principals","id":"${user.name}","path":"principals"}}}
+ ],"minimum_should_match":1}}
diff --git a/cloud_securityconfig/roles_mapping.yml b/cloud_securityconfig/roles_mapping.yml
index 8a0025462..3634cccae 100644
--- a/cloud_securityconfig/roles_mapping.yml
+++ b/cloud_securityconfig/roles_mapping.yml
@@ -8,6 +8,12 @@ openrag_user_role:
backend_roles:
- "openrag_user"
+openrag_user_acl_role:
+ users: []
+ hosts: []
+ backend_roles:
+ - "openrag_user"
+
all_access:
users:
- "admin"
diff --git a/docker-compose.yml b/docker-compose.yml
index 8c6c963c2..9a9cd015a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -35,7 +35,7 @@ services:
restart: unless-stopped
dashboards:
- image: docker.io/opensearchproject/opensearch-dashboards:3.6.0
+ image: docker.io/opensearchproject/opensearch-dashboards:3.8.0
container_name: ${COMPOSE_PROJECT_NAME:-openrag}-dashboards
depends_on:
- opensearch
@@ -355,4 +355,4 @@ services:
volumes:
opensearch-data:
- azurite-data:
\ No newline at end of file
+ azurite-data:
diff --git a/docs/docs/get-started/docker.mdx b/docs/docs/get-started/docker.mdx
index e78ebaab2..4bfcd730a 100644
--- a/docs/docs/get-started/docker.mdx
+++ b/docs/docs/get-started/docker.mdx
@@ -139,10 +139,10 @@ This is only supported for self-managed deployments. For Kubernetes deployments,
| `docker.io/langflowai/openrag-frontend:VERSION` | `REGISTRY/ORG/openrag-frontend:TAG` |
| `docker.io/langflowai/openrag-langflow:VERSION` | `REGISTRY/ORG/openrag-langflow:TAG` |
| `docker.io/langflowai/openrag-opensearch:VERSION` | `REGISTRY/ORG/openrag-opensearch:TAG` |
- | `docker.io/opensearchproject/opensearch-dashboards:3.6.0` | `DASHBOARDS_REGISTRY/opensearchproject/opensearch-dashboards:3.6.0` |
+ | `docker.io/opensearchproject/opensearch-dashboards:3.8.0` | `DASHBOARDS_REGISTRY/opensearchproject/opensearch-dashboards:3.8.0` |
The first four images resolve from `IMAGE_REGISTRY` and `IMAGE_ORG`, so you can mirror them into any organization you control.
- The OpenSearch Dashboards image resolves from `DASHBOARDS_REGISTRY` only, so you must preserve the `opensearchproject/opensearch-dashboards` path and the `3.6.0` tag when you mirror it.
+ The OpenSearch Dashboards image resolves from `DASHBOARDS_REGISTRY` only, so you must preserve the `opensearchproject/opensearch-dashboards` path and the `3.8.0` tag when you mirror it.
:::tip
The OpenRAG images are multi-architecture manifests.
@@ -275,4 +275,4 @@ If the containers fail to start with an image pull error, see [Container image p
<PartialOnboarding />
-<PartialInstallNextSteps />
\ No newline at end of file
+<PartialInstallNextSteps />
diff --git a/docs/docs/reference/configuration.mdx b/docs/docs/reference/configuration.mdx
index 5828a918d..04f66e8e6 100644
--- a/docs/docs/reference/configuration.mdx
+++ b/docs/docs/reference/configuration.mdx
@@ -317,7 +317,7 @@ For a complete walkthrough, including how to mirror the images, see [Deploy Open
| `IMAGE_REGISTRY` | `docker.io` | Registry host for the OpenRAG-owned images (`openrag-backend`, `openrag-frontend`, `openrag-langflow`, and `openrag-opensearch`). Set this to your registry hostname, and include the port if your registry uses a non-standard one, such as `registry.example.com:5000`. Don't include a scheme (`https://`) or a trailing slash. This variable doesn't affect the OpenSearch Dashboards image; use `DASHBOARDS_REGISTRY` for that image. |
| `IMAGE_ORG` | `langflowai` | Organization or namespace that contains the OpenRAG-owned images within `IMAGE_REGISTRY`. Together, these variables form the repository path `IMAGE_REGISTRY/IMAGE_ORG/IMAGE_NAME`. For example, `IMAGE_REGISTRY=registry.example.com` and `IMAGE_ORG=platform-team` resolves the backend image to `registry.example.com/platform-team/openrag-backend`. |
| `OPENRAG_IMAGE_TAG` | Not set | Overrides the image tag applied to all OpenRAG-owned images. When set, this variable takes precedence over `OPENRAG_VERSION`. When it isn't set, OpenRAG falls back to `OPENRAG_VERSION`, and then to `latest`. Use this variable when your mirror re-tags images and the tags don't match the OpenRAG release versions. |
-| `DASHBOARDS_REGISTRY` | `docker.io` | Registry host for the OpenSearch Dashboards image only. This lets you point Dashboards at a mirror without redirecting the OpenRAG-owned images. The repository path and tag within the registry are fixed, so the mirrored image must be available at `DASHBOARDS_REGISTRY/opensearchproject/opensearch-dashboards:3.6.0`. |
+| `DASHBOARDS_REGISTRY` | `docker.io` | Registry host for the OpenSearch Dashboards image only. This lets you point Dashboards at a mirror without redirecting the OpenRAG-owned images. The repository path and tag within the registry are fixed, so the mirrored image must be available at `DASHBOARDS_REGISTRY/opensearchproject/opensearch-dashboards:3.8.0`. |
:::important
These variables are read by Docker Compose, so they apply to [self-managed deployments](/docker).
diff --git a/docs/docs/support/troubleshoot.mdx b/docs/docs/support/troubleshoot.mdx
index ced0e976f..3f95c576f 100644
--- a/docs/docs/support/troubleshoot.mdx
+++ b/docs/docs/support/troubleshoot.mdx
@@ -53,7 +53,7 @@ The reason identifies which part of your [image and registry configuration](/ref
This check only covers the OpenRAG-owned images.
The OpenSearch Dashboards image isn't validated in advance, so an incorrect `DASHBOARDS_REGISTRY` value doesn't produce one of the errors above.
Instead, the deployment fails later with a pull error from Docker or Podman.
-Because the repository path and tag for that image are fixed, confirm that the mirrored image is available at `DASHBOARDS_REGISTRY/opensearchproject/opensearch-dashboards:3.6.0`, and make sure `DASHBOARDS_REGISTRY` is set correctly.
+Because the repository path and tag for that image are fixed, confirm that the mirrored image is available at `DASHBOARDS_REGISTRY/opensearchproject/opensearch-dashboards:3.8.0`, and make sure `DASHBOARDS_REGISTRY` is set correctly.
For instructions on mirroring the images, see [Deploy OpenRAG from a private registry](/docker#private-registry).
@@ -458,4 +458,4 @@ The following steps explain how to troubleshoot these issues with Fedora 43, Pod
```
12. In a terminal, run OpenRAG, and then start the OpenRAG services with GPU mode enabled.
-Or, for self-managed deployments, deploy OpenRAG with the `docker-compose-gpu.yml` file.
\ No newline at end of file
+Or, for self-managed deployments, deploy OpenRAG with the `docker-compose-gpu.yml` file.
diff --git a/kubernetes/helm/openrag/values.yaml b/kubernetes/helm/openrag/values.yaml
index a4eb94b7a..15a8ffd7a 100644
--- a/kubernetes/helm/openrag/values.yaml
+++ b/kubernetes/helm/openrag/values.yaml
@@ -482,7 +482,7 @@ dashboards:
image:
repository: opensearchproject/opensearch-dashboards
- tag: "3.6.0"
+ tag: "3.8.0"
replicaCount: 1
diff --git a/securityconfig/roles.yml b/securityconfig/roles.yml
index 01f697787..a918b96d4 100644
--- a/securityconfig/roles.yml
+++ b/securityconfig/roles.yml
@@ -3,7 +3,7 @@ _meta:
config_version: 2
openrag_user_role:
- description: "DLS: user can read docs they own or are allowed on"
+ description: "DLS: user can read docs they own or unowned docs"
cluster_permissions:
- "indices:data/read/scroll"
- "indices:data/read/scroll/clear"
@@ -17,9 +17,6 @@ openrag_user_role:
{"bool":{"should":[
{"term":{"owner":"${user.name}"}},
{"term":{"owner":"${attr.jwt.email}"}},
- {"term":{"allowed_users":"${user.name}"}},
- {"term":{"allowed_users":"${attr.jwt.email}"}},
- {"terms":{"allowed_principals":{"index":"openrag_dls_principals","id":"${user.name}","path":"principals"}}},
{"bool":{"must_not":{"exists":{"field":"owner"}}}}
],"minimum_should_match":1}}
- index_patterns: ["documents", "*documents*", "knowledge_filters", "knowledge_filters*"]
@@ -50,3 +47,18 @@ openrag_user_role:
- indices:admin/mappings/get
- indices:admin/exists
- indices:admin/get
+
+openrag_user_acl_role:
+ description: "DLS: user can read docs they are allowed on via ACL"
+ cluster_permissions: []
+
+ index_permissions:
+ - index_patterns: ["documents", "*documents*", "knowledge_filters", "knowledge_filters*"]
+ allowed_actions:
+ - read
+ dls: >
+ {"bool":{"should":[
+ {"term":{"allowed_users":"${user.name}"}},
+ {"term":{"allowed_users":"${attr.jwt.email}"}},
+ {"terms":{"allowed_principals":{"index":"openrag_dls_principals","id":"${user.name}","path":"principals"}}}
+ ],"minimum_should_match":1}}
diff --git a/securityconfig/roles_mapping.yml b/securityconfig/roles_mapping.yml
index f22717d99..8fa34a5dd 100644
--- a/securityconfig/roles_mapping.yml
+++ b/securityconfig/roles_mapping.yml
@@ -8,6 +8,12 @@ openrag_user_role:
backend_roles:
- "openrag_user"
+openrag_user_acl_role:
+ users: []
+ hosts: []
+ backend_roles:
+ - "openrag_user"
+
all_access:
users:
- "admin"
diff --git a/src/utils/opensearch_utils.py b/src/utils/opensearch_utils.py
index 1c5c5e20f..4dcde03b2 100644
--- a/src/utils/opensearch_utils.py
+++ b/src/utils/opensearch_utils.py
@@ -1,4 +1,5 @@
import asyncio
+import copy
import os
import random
from typing import Any
@@ -10,6 +11,11 @@ from utils.logging_config import get_logger
logger = get_logger(__name__)
+OPENRAG_USER_ROLE = "openrag_user_role"
+OPENRAG_USER_ACL_ROLE = "openrag_user_acl_role"
+ALL_ACCESS_ROLE = "all_access"
+ROLE_MAPPING_PRINCIPAL_FIELDS = {"users", "hosts", "backend_roles", "and_backend_roles"}
+
DISK_SPACE_ERROR_MESSAGE = (
"OpenSearch has run out of available disk space. "
"Search and indexing operations are blocked. "
@@ -52,6 +58,50 @@ AUTH_ERROR_MESSAGE = (
"Authentication failed: OpenSearch rejected the credential. Please sign in again."
)
+_SECURITY_SETUP_SKIP_INDICATORS = [
+ "401",
+ "403",
+ "404",
+ "security_exception",
+ "not_found",
+ "unauthorized",
+ "forbidden",
+]
+
+
+def _is_security_setup_skip_error(error: Exception) -> bool:
+ """True when the security API is unavailable before any mutation is attempted."""
+ error_str = str(error).lower()
+ return any(indicator in error_str for indicator in _SECURITY_SETUP_SKIP_INDICATORS)
+
+
+def _is_not_found_error(error: Exception) -> bool:
+ """True when OpenSearch reports a missing security object."""
+ error_str = str(error).lower()
+ return "404" in error_str or "not_found" in error_str
+
+
+def _dedupe_preserving_order(values: list[Any]) -> list[Any]:
+ """Return unique values while preserving their first-seen order."""
+ result: list[Any] = []
+ for value in values:
+ if value not in result:
+ result.append(value)
+ return result
+
+
+def _merge_role_mappings(*mappings: dict[str, Any] | None) -> dict[str, Any]:
+ """Merge OpenSearch role-mapping principals without dropping custom entries."""
+ merged: dict[str, Any] = {}
+ for mapping in mappings:
+ if not mapping:
+ continue
+ for key, value in mapping.items():
+ if key not in ROLE_MAPPING_PRINCIPAL_FIELDS:
+ continue
+ merged[key] = _dedupe_preserving_order((merged.get(key) or []) + (value or []))
+ return merged
+
def is_opensearch_auth_error(error: Exception | str) -> bool:
"""Whether *error* is an OpenSearch authentication failure (401).
@@ -294,10 +344,10 @@ async def setup_opensearch_security(
The setup involves:
1. GET /_plugins/_security/api/rolesmapping (check existing)
2. GET /_cluster/health
- 3. PUT /_plugins/_security/api/roles/openrag_user_role (create role)
- 4. PUT /_plugins/_security/api/rolesmapping/openrag_user_role (create mapping)
- 5. PUT /_plugins/_security/api/rolesmapping/all_access (merge admin mapping)
- 6. Verify with final GETs.
+ 3. PUT and verify additive roles/mappings, including openrag_user_acl_role.
+ 4. PUT narrowed legacy roles/mappings, preserving existing mapping principals.
+ 5. PUT /_plugins/_security/api/rolesmapping/all_access (merge admin mapping).
+ 6. Verify updated legacy roles/mappings with final GETs.
Args:
opensearch_client: Authenticated OpenSearch client.
@@ -339,6 +389,7 @@ async def setup_opensearch_security(
# 1. & 2. Readiness checks
logger.info("[OPENSEARCH] Performing readiness checks...")
+ rolesmapping_response: dict[str, Any] = {}
try:
rolesmapping_response = await opensearch_client.transport.perform_request(
"GET", "/_plugins/_security/api/rolesmapping"
@@ -350,7 +401,15 @@ async def setup_opensearch_security(
else "unknown",
)
except Exception as e:
- logger.warning("[OPENSEARCH] Failed to get current rolesmapping", error=str(e))
+ if _is_security_setup_skip_error(e):
+ logger.warning(
+ "Skipping OpenSearch security configuration: "
+ "The cluster may not have the security plugin enabled or "
+ "the provided credentials do not have administrative permissions."
+ )
+ return
+ logger.error("[OPENSEARCH] Failed to get current rolesmapping", error=str(e))
+ raise
cluster_health = await opensearch_client.cluster.health()
logger.info("[OPENSEARCH] Cluster health check passed", status=cluster_health.get("status"))
@@ -368,12 +427,53 @@ async def setup_opensearch_security(
roles=list(roles_config.keys()) if roles_config else [],
)
- # 3. Create openrag_user_role
- if "openrag_user_role" in roles_config:
- role_body = roles_config["openrag_user_role"]
+ # Load roles mapping from YAML
+ if not os.path.exists(roles_mapping_file):
+ logger.error(f"[OPENSEARCH] Roles mapping file not found: {roles_mapping_file}")
+ raise FileNotFoundError(f"Roles mapping file not found: {roles_mapping_file}")
+
+ with open(roles_mapping_file) as f:
+ mapping_config = yaml.safe_load(f)
+
+ logger.info(
+ "[OPENSEARCH] Loaded roles mapping configuration",
+ mappings=list(mapping_config.keys()) if mapping_config else [],
+ )
+ role_names = [role_name for role_name in roles_config if not role_name.startswith("_")]
+ pre_legacy_role_names = [
+ role_name for role_name in role_names if role_name != OPENRAG_USER_ROLE
+ ]
+ ordered_role_names = pre_legacy_role_names + [
+ role_name for role_name in role_names if role_name == OPENRAG_USER_ROLE
+ ]
+
+ mapping_names = [
+ mapping_name
+ for mapping_name in mapping_config
+ if not mapping_name.startswith("_") and mapping_name != ALL_ACCESS_ROLE
+ ]
+ pre_legacy_mapping_names = [
+ mapping_name for mapping_name in mapping_names if mapping_name != OPENRAG_USER_ROLE
+ ]
+ ordered_mapping_names = pre_legacy_mapping_names + [
+ mapping_name for mapping_name in mapping_names if mapping_name == OPENRAG_USER_ROLE
+ ]
+
+ def merged_mapping_body(mapping_name: str) -> dict[str, Any]:
+ configured = copy.deepcopy(mapping_config[mapping_name])
+ existing_same = rolesmapping_response.get(mapping_name, {})
+ if mapping_name == OPENRAG_USER_ACL_ROLE:
+ existing_legacy = rolesmapping_response.get(OPENRAG_USER_ROLE, {})
+ return _merge_role_mappings(configured, existing_legacy, existing_same)
+ if mapping_name == OPENRAG_USER_ROLE:
+ return _merge_role_mappings(configured, existing_same)
+ return _merge_role_mappings(configured, existing_same)
+
+ async def put_role(role_name: str) -> None:
+ role_body = roles_config[role_name]
logger.info(
- "[OPENSEARCH] Creating 'openrag_user_role' role",
+ f"[OPENSEARCH] Creating/updating '{role_name}' role",
patterns=role_body["index_permissions"][0]["index_patterns"]
if "index_permissions" in role_body
else "default",
@@ -384,49 +484,73 @@ async def setup_opensearch_security(
resp = await opensearch_client.transport.perform_request(
"PUT",
- "/_plugins/_security/api/roles/openrag_user_role",
+ f"/_plugins/_security/api/roles/{role_name}",
body=role_body,
headers={"Content-Type": "application/json"},
)
- logger.info("[OPENSEARCH] Role creation response", response=resp)
- else:
- logger.warning("[OPENSEARCH] 'openrag_user_role' not found in roles.yml")
+ logger.info(f"[OPENSEARCH] Role '{role_name}' creation response", response=resp)
- # Load roles mapping from YAML
- if not os.path.exists(roles_mapping_file):
- logger.error(f"[OPENSEARCH] Roles mapping file not found: {roles_mapping_file}")
- raise FileNotFoundError(f"Roles mapping file not found: {roles_mapping_file}")
-
- with open(roles_mapping_file) as f:
- mapping_config = yaml.safe_load(f)
-
- logger.info(
- "[OPENSEARCH] Loaded roles mapping configuration",
- mappings=list(mapping_config.keys()) if mapping_config else [],
- )
+ async def verify_role(role_name: str) -> None:
+ role_verify = await opensearch_client.transport.perform_request(
+ "GET", f"/_plugins/_security/api/roles/{role_name}"
+ )
+ logger.info("[OPENSEARCH] Role verification", role_name=role_name, role=role_verify)
- # 4. Create openrag_user_role mapping
- if "openrag_user_role" in mapping_config:
- mapping_body = mapping_config["openrag_user_role"]
+ async def put_mapping(mapping_name: str) -> None:
+ mapping_body = merged_mapping_body(mapping_name)
logger.info(
- "[OPENSEARCH] Creating 'openrag_user_role' mapping",
+ f"[OPENSEARCH] Creating/updating '{mapping_name}' mapping",
backend_roles=mapping_body.get("backend_roles", []),
users=mapping_body.get("users", []),
+ hosts=mapping_body.get("hosts", []),
)
resp = await opensearch_client.transport.perform_request(
"PUT",
- "/_plugins/_security/api/rolesmapping/openrag_user_role",
+ f"/_plugins/_security/api/rolesmapping/{mapping_name}",
body=mapping_body,
headers={"Content-Type": "application/json"},
)
- logger.info("[OPENSEARCH] Role mapping update response", response=resp)
+ logger.info(
+ f"[OPENSEARCH] Role mapping '{mapping_name}' update response", response=resp
+ )
+
+ async def verify_mapping(mapping_name: str) -> None:
+ mapping_verify = await opensearch_client.transport.perform_request(
+ "GET", f"/_plugins/_security/api/rolesmapping/{mapping_name}"
+ )
+ logger.info(
+ "[OPENSEARCH] Role mapping verification",
+ mapping_name=mapping_name,
+ mapping=mapping_verify,
+ )
+
+ # Apply and verify the additive ACL role/mapping before narrowing the
+ # legacy role. This avoids losing shared-document access if a later
+ # security API call fails during upgrade.
+ for role_name in pre_legacy_role_names:
+ await put_role(role_name)
+ for mapping_name in pre_legacy_mapping_names:
+ await put_mapping(mapping_name)
+ for role_name in pre_legacy_role_names:
+ await verify_role(role_name)
+ for mapping_name in pre_legacy_mapping_names:
+ await verify_mapping(mapping_name)
+
+ for role_name in ordered_role_names:
+ if role_name in pre_legacy_role_names:
+ continue
+ await put_role(role_name)
+ for mapping_name in ordered_mapping_names:
+ if mapping_name in pre_legacy_mapping_names:
+ continue
+ await put_mapping(mapping_name)
# 5. Update all_access mapping — merge with existing to preserve
# IBM-managed entries, but ensure backend_roles never contains
# "all_access" (which would give IBM API key users the super-admin
# role and bypass DLS).
- if "all_access" in mapping_config:
- all_access_body = mapping_config["all_access"]
+ if ALL_ACCESS_ROLE in mapping_config:
+ all_access_body = copy.deepcopy(mapping_config[ALL_ACCESS_ROLE])
if "backend_roles" not in all_access_body:
all_access_body["backend_roles"] = ["admin"]
@@ -442,11 +566,13 @@ async def setup_opensearch_security(
existing = await opensearch_client.transport.perform_request(
"GET", "/_plugins/_security/api/rolesmapping/all_access"
)
- existing_mapping = existing.get("all_access", {})
+ existing_mapping = existing.get(ALL_ACCESS_ROLE, {})
existing_users = existing_mapping.get("users", []) or []
existing_hosts = existing_mapping.get("hosts", []) or []
existing_backend_roles = existing_mapping.get("backend_roles", []) or []
- except Exception:
+ except Exception as e:
+ if not _is_not_found_error(e):
+ raise
logger.debug("[OPENSEARCH] No existing all_access mapping found, creating fresh")
# Build merged users: source file + cluster + new admin (bare + ibmlhapikey_ variant).
@@ -461,14 +587,16 @@ async def setup_opensearch_security(
users=new_admin_users,
)
- merged_users = list(
- set(all_access_body.get("users", []) + existing_users + new_admin_users)
+ merged_users = _dedupe_preserving_order(
+ all_access_body.get("users", []) + existing_users + new_admin_users
)
all_access_body["users"] = merged_users
logger.debug("[OPENSEARCH] Merged all_access users", users=merged_users)
if existing_hosts:
- merged_hosts = list(set(all_access_body.get("hosts", []) + existing_hosts))
+ merged_hosts = _dedupe_preserving_order(
+ all_access_body.get("hosts", []) + existing_hosts
+ )
all_access_body["hosts"] = merged_hosts
logger.debug(
"[OPENSEARCH] Preserved existing all_access hosts",
@@ -479,8 +607,8 @@ async def setup_opensearch_security(
safe_existing_backend_roles = [
r for r in existing_backend_roles if r != "all_access"
]
- merged_backend_roles = list(
- set(all_access_body.get("backend_roles", []) + safe_existing_backend_roles)
+ merged_backend_roles = _dedupe_preserving_order(
+ all_access_body.get("backend_roles", []) + safe_existing_backend_roles
)
all_access_body["backend_roles"] = merged_backend_roles
logger.debug(
@@ -508,31 +636,17 @@ async def setup_opensearch_security(
# 6. Final verification
logger.info("[OPENSEARCH] Verifying security configuration...")
- role_verify = await opensearch_client.transport.perform_request(
- "GET", "/_plugins/_security/api/roles/openrag_user_role"
- )
- logger.info("[OPENSEARCH] Role verification", role=role_verify)
-
- mapping_verify = await opensearch_client.transport.perform_request(
- "GET", "/_plugins/_security/api/rolesmapping/openrag_user_role"
- )
- logger.info("[OPENSEARCH] Role mapping verification", mapping=mapping_verify)
+ for role_name in ordered_role_names:
+ if role_name in pre_legacy_role_names:
+ continue
+ await verify_role(role_name)
+ for mapping_name in ordered_mapping_names:
+ if mapping_name in pre_legacy_mapping_names:
+ continue
+ await verify_mapping(mapping_name)
logger.info("Successfully completed OpenSearch security configuration.")
except Exception as e:
- # Check for authentication errors or if the security plugin is missing
- error_str = str(e).lower()
- if any(
- code in error_str for code in ["401", "403", "404", "security_exception", "not_found"]
- ):
- logger.warning(
- "Skipping OpenSearch security configuration: "
- "The cluster may not have the security plugin enabled or "
- "the provided credentials do not have administrative permissions."
- )
- return
-
logger.error("Failed to setup OpenSearch security configuration", error=str(e))
- # Re-raise for non-auth/non-security errors to ensure visibility
raise
diff --git a/tests/unit/test_group_acl.py b/tests/unit/test_group_acl.py
index 3af6235ae..0b599e58d 100644
--- a/tests/unit/test_group_acl.py
+++ b/tests/unit/test_group_acl.py
@@ -188,29 +188,56 @@ def test_group_acl_service_invalidation_drops_cache_and_locks():
def test_security_roles_include_acl_dls_queries():
for rel_path in ("securityconfig/roles.yml", "cloud_securityconfig/roles.yml"):
roles = yaml.safe_load((ROOT / rel_path).read_text())
- index_permissions = roles["openrag_user_role"]["index_permissions"]
+ assert "openrag_user_acl_role" in roles
+
+ user_role = roles["openrag_user_role"]
+ acl_role = roles["openrag_user_acl_role"]
+ index_permissions = [
+ p for role in (user_role, acl_role) for p in role.get("index_permissions", [])
+ ]
cluster_permissions = roles["openrag_user_role"]["cluster_permissions"]
assert "indices:data/write/bulk" not in cluster_permissions
assert "indices:data/write/index" not in cluster_permissions
assert not any("alerting" in permission for permission in cluster_permissions)
-
- document_permission = index_permissions[0]
- document_actions = document_permission["allowed_actions"]
- assert "read" in document_actions
- assert "crud" not in document_actions
- assert "indices:data/write/index" not in document_actions
- assert "indices:data/write/update/byquery" not in document_actions
- assert "indices:admin/mappings/put" not in document_actions
-
- dls = index_permissions[0]["dls"]
- assert '{"term":{"owner":"${user.name}"}}' in dls
- assert '{"term":{"owner":"${attr.jwt.email}"}}' in dls
- assert '{"term":{"allowed_users":"${user.name}"}}' in dls
- assert '{"term":{"allowed_users":"${attr.jwt.email}"}}' in dls
- assert '{"terms":{"allowed_groups":[${user.roles}]}}' not in dls
+ assert acl_role["cluster_permissions"] == []
+
+ legacy_document_permissions = [
+ p
+ for p in user_role.get("index_permissions", [])
+ if any("documents" in pattern for pattern in p.get("index_patterns", []))
+ and "read" in p.get("allowed_actions", [])
+ ]
+ acl_document_permissions = [
+ p
+ for p in acl_role.get("index_permissions", [])
+ if any("documents" in pattern for pattern in p.get("index_patterns", []))
+ and "read" in p.get("allowed_actions", [])
+ ]
+ assert len(legacy_document_permissions) == 1
+ assert len(acl_document_permissions) == 1
+
+ document_permissions = legacy_document_permissions + acl_document_permissions
+ for document_permission in document_permissions:
+ document_actions = document_permission["allowed_actions"]
+ assert "read" in document_actions
+ assert "crud" not in document_actions
+ assert "indices:data/write/index" not in document_actions
+ assert "indices:data/write/update/byquery" not in document_actions
+ assert "indices:admin/mappings/put" not in document_actions
+ assert len("".join(document_permission["dls"].split())) <= 256
+
+ legacy_dls = "".join(legacy_document_permissions[0]["dls"].split())
+ acl_dls = "".join(acl_document_permissions[0]["dls"].split())
+ assert '{"term":{"owner":"${user.name}"}}' in legacy_dls
+ assert '{"term":{"owner":"${attr.jwt.email}"}}' in legacy_dls
+ assert '{"term":{"allowed_users":"${user.name}"}}' not in legacy_dls
+ assert '{"term":{"allowed_users":"${user.name}"}}' in acl_dls
+ assert '{"term":{"allowed_users":"${attr.jwt.email}"}}' in acl_dls
+ assert '{"term":{"owner":"${user.name}"}}' not in acl_dls
+ assert '{"terms":{"allowed_groups":[${user.roles}]}}' not in acl_dls
assert (
'{"terms":{"allowed_principals":{"index":"openrag_dls_principals",'
- '"id":"${user.name}","path":"principals"}}}' in dls
+ '"id":"${user.name}","path":"principals"}}}' in acl_dls
)
principal_permission = next(
permission
diff --git a/tests/unit/test_opensearch_security_setup.py b/tests/unit/test_opensearch_security_setup.py
index fa3ea60ee..70fa4a145 100644
--- a/tests/unit/test_opensearch_security_setup.py
+++ b/tests/unit/test_opensearch_security_setup.py
@@ -1,69 +1,203 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+
import pytest
-from unittest.mock import AsyncMock, patch, MagicMock
from utils.opensearch_utils import setup_opensearch_security
-async def test_setup_opensearch_security_success():
- """Test successful security setup with all expected calls."""
- mock_client = MagicMock()
- mock_client.transport.perform_request = AsyncMock(return_value={"status": "OK", "message": "Success"})
- mock_client.cluster.health = AsyncMock(return_value={"status": "green"})
- # Sample configurations
- roles_data = {
+def _sample_roles():
+ return {
"openrag_user_role": {
- "cluster_permissions": ["read"],
- "index_permissions": [{"index_patterns": ["*"], "allowed_actions": ["crud"]}]
- }
+ "cluster_permissions": ["cluster:monitor/*"],
+ "index_permissions": [
+ {
+ "index_patterns": ["documents"],
+ "allowed_actions": ["read"],
+ "dls": '{"term":{"owner":"${user.name}"}}',
+ }
+ ],
+ },
+ "openrag_user_acl_role": {
+ "cluster_permissions": [],
+ "index_permissions": [
+ {
+ "index_patterns": ["documents"],
+ "allowed_actions": ["read"],
+ "dls": '{"term":{"allowed_users":"${user.name}"}}',
+ }
+ ],
+ },
}
- mapping_data = {
- "openrag_user_role": {"backend_roles": ["openrag_user"]},
- "all_access": {"users": ["admin"]}
+
+
+def _sample_mappings():
+ return {
+ "openrag_user_role": {
+ "users": [],
+ "hosts": [],
+ "backend_roles": ["openrag_user"],
+ },
+ "openrag_user_acl_role": {
+ "users": [],
+ "hosts": [],
+ "backend_roles": ["openrag_user"],
+ },
+ "all_access": {"users": ["admin"]},
}
- # Mock file existence and content
- with patch("os.path.exists", return_value=True), \
- patch("builtins.open", MagicMock()), \
- patch("yaml.safe_load") as mock_yaml, \
- patch("utils.opensearch_utils.get_index_name", return_value="dynamic_index"):
-
- mock_yaml.side_effect = [roles_data, mapping_data]
+async def test_setup_opensearch_security_applies_acl_role_before_narrowing_legacy_role():
+ """ACL role and mapping are verified before the legacy role is narrowed."""
+ mock_client = MagicMock()
+ requests = []
+ existing_mappings = {
+ "openrag_user_role": {
+ "users": ["direct-user"],
+ "hosts": ["legacy-host"],
+ "backend_roles": ["legacy_backend_role"],
+ "and_backend_roles": ["legacy_and_role"],
+ "hidden": False,
+ "reserved": False,
+ },
+ "openrag_user_acl_role": {
+ "users": ["acl-user"],
+ "hosts": [],
+ "backend_roles": ["acl_backend_role"],
+ "hidden": False,
+ "reserved": False,
+ },
+ }
+
+ async def perform_request(method, path, **kwargs):
+ requests.append((method, path, kwargs.get("body")))
+ if method == "GET" and path == "/_plugins/_security/api/rolesmapping":
+ return existing_mappings
+ if method == "GET" and path == "/_plugins/_security/api/rolesmapping/all_access":
+ return {"all_access": {"users": ["existing-admin"], "backend_roles": ["admin"]}}
+ return {"status": "OK", "message": "Success"}
+
+ mock_client.transport.perform_request = AsyncMock(side_effect=perform_request)
+ mock_client.cluster.health = AsyncMock(return_value={"status": "green"})
+
+ with (
+ patch("os.path.exists", return_value=True),
+ patch("builtins.open", MagicMock()),
+ patch("yaml.safe_load", side_effect=[_sample_roles(), _sample_mappings()]),
+ ):
await setup_opensearch_security(mock_client)
- # Verify calls
- assert mock_client.transport.perform_request.call_count == 6
- mock_client.cluster.health.assert_called_once()
+ request_paths = [(method, path) for method, path, _ in requests]
+ assert request_paths.index(
+ ("PUT", "/_plugins/_security/api/roles/openrag_user_acl_role")
+ ) < request_paths.index(("PUT", "/_plugins/_security/api/roles/openrag_user_role"))
+ assert request_paths.index(
+ ("PUT", "/_plugins/_security/api/rolesmapping/openrag_user_acl_role")
+ ) < request_paths.index(("PUT", "/_plugins/_security/api/roles/openrag_user_role"))
+ assert request_paths.index(
+ ("GET", "/_plugins/_security/api/roles/openrag_user_acl_role")
+ ) < request_paths.index(("PUT", "/_plugins/_security/api/roles/openrag_user_role"))
+ assert request_paths.index(
+ ("GET", "/_plugins/_security/api/rolesmapping/openrag_user_acl_role")
+ ) < request_paths.index(("PUT", "/_plugins/_security/api/roles/openrag_user_role"))
+
+ acl_mapping_put = next(
+ body
+ for method, path, body in requests
+ if method == "PUT" and path.endswith("/rolesmapping/openrag_user_acl_role")
+ )
+ assert acl_mapping_put["users"] == ["direct-user", "acl-user"]
+ assert acl_mapping_put["hosts"] == ["legacy-host"]
+ assert acl_mapping_put["backend_roles"] == [
+ "openrag_user",
+ "legacy_backend_role",
+ "acl_backend_role",
+ ]
+ assert acl_mapping_put["and_backend_roles"] == ["legacy_and_role"]
+ assert "hidden" not in acl_mapping_put
+ assert "reserved" not in acl_mapping_put
+
+ legacy_mapping_put = next(
+ body
+ for method, path, body in requests
+ if method == "PUT" and path.endswith("/rolesmapping/openrag_user_role")
+ )
+ assert legacy_mapping_put["users"] == ["direct-user"]
+ assert legacy_mapping_put["hosts"] == ["legacy-host"]
+ assert legacy_mapping_put["backend_roles"] == [
+ "openrag_user",
+ "legacy_backend_role",
+ ]
+ assert legacy_mapping_put["and_backend_roles"] == ["legacy_and_role"]
+ assert "hidden" not in legacy_mapping_put
+ assert "reserved" not in legacy_mapping_put
- # Check the role creation body for dynamic patterns
- role_put_call = mock_client.transport.perform_request.call_args_list[1]
- role_body = role_put_call[1]['body']
- patterns = role_body['index_permissions'][0]['index_patterns']
- assert "dynamic_index" in patterns
- assert "dynamic_index*" in patterns
- assert "knowledge_filters" in patterns
- assert "knowledge_filters*" in patterns
@pytest.mark.asyncio
-async def test_setup_opensearch_security_graceful_auth_error():
- """Test that auth/security errors are handled gracefully without raising."""
+async def test_setup_opensearch_security_skips_auth_error_before_mutation():
+ """Auth/security errors are graceful only before any security state is mutated."""
mock_client = MagicMock()
- # Mock a 401 Unauthorized error
mock_client.transport.perform_request = AsyncMock(side_effect=Exception("401 Unauthorized"))
-
- # This should NOT raise an exception
+ mock_client.cluster.health = AsyncMock()
+
await setup_opensearch_security(mock_client)
+
assert mock_client.transport.perform_request.call_count == 1
+ mock_client.cluster.health.assert_not_called()
+
+
+async def test_setup_opensearch_security_aborts_on_transient_mapping_read_failure():
+ """Without current mappings, setup cannot safely preserve customized principals."""
+ mock_client = MagicMock()
+ mock_client.transport.perform_request = AsyncMock(
+ side_effect=Exception("503 Service Unavailable")
+ )
+ mock_client.cluster.health = AsyncMock()
+
+ with pytest.raises(Exception, match="503 Service Unavailable"):
+ await setup_opensearch_security(mock_client)
+
+ mock_client.transport.perform_request.assert_awaited_once_with(
+ "GET", "/_plugins/_security/api/rolesmapping"
+ )
+ mock_client.cluster.health.assert_not_called()
+
+
+async def test_setup_opensearch_security_raises_security_error_after_partial_migration():
+ """A post-ACL failure must fail startup instead of hiding partial security state."""
+ mock_client = MagicMock()
+ requests = []
+
+ async def perform_request(method, path, **kwargs):
+ requests.append((method, path))
+ if method == "PUT" and path == "/_plugins/_security/api/roles/openrag_user_role":
+ raise Exception("403 forbidden")
+ return {}
+
+ mock_client.transport.perform_request = AsyncMock(side_effect=perform_request)
+ mock_client.cluster.health = AsyncMock(return_value={"status": "green"})
+
+ with (
+ patch("os.path.exists", return_value=True),
+ patch("builtins.open", MagicMock()),
+ patch("yaml.safe_load", side_effect=[_sample_roles(), _sample_mappings()]),
+ ):
+ with pytest.raises(Exception, match="403 forbidden"):
+ await setup_opensearch_security(mock_client)
+
+ assert ("PUT", "/_plugins/_security/api/rolesmapping/openrag_user_acl_role") in requests
+ assert ("GET", "/_plugins/_security/api/rolesmapping/openrag_user_acl_role") in requests
+
@pytest.mark.asyncio
async def test_setup_opensearch_security_missing_files():
"""Test that missing configuration files raise FileNotFoundError."""
mock_client = MagicMock()
- mock_client.transport.perform_request = AsyncMock()
- mock_client.cluster.health = AsyncMock()
-
- with patch("os.path.exists", return_value=False), \
- patch("utils.opensearch_utils.get_index_name", return_value="docs"):
+ mock_client.transport.perform_request = AsyncMock(return_value={})
+ mock_client.cluster.health = AsyncMock(return_value={"status": "green"})
+
+ with patch("os.path.exists", return_value=False):
with pytest.raises(FileNotFoundError):
await setup_opensearch_security(mock_client)