OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: upgrade OpenSearch to 3.6.0 (#1972)
Lucas Oliveira74fe6bec1aad
diff --git a/Dockerfile b/Dockerfileindex b8911ae35..6b8e18154 100644--- a/Dockerfile+++ b/Dockerfile@@ -1,31 +1,29 @@######################################### Stage 1: Upstream OpenSearch with plugins########################################-FROM opensearchproject/opensearch:3.2.0 AS upstream_opensearch+FROM opensearchproject/opensearch:3.6.0 AS upstream_opensearch# Remove pluginsRUN opensearch-plugin remove opensearch-neural-search || true && \- opensearch-plugin remove opensearch-knn || true && \- # removing this one due to Netty CVE-2025-58056, can bring it back in the future- opensearch-plugin remove opensearch-security-analytics || true+ opensearch-plugin remove opensearch-knn || true# Prepare jvector plugin artifactsRUN mkdir -p /tmp/opensearch-jvector-plugin && \- curl -L -s https://github.com/opensearch-project/opensearch-jvector/releases/download/3.2.0.0/artifacts.tar.gz \+ curl -L -s https://github.com/opensearch-project/opensearch-jvector/releases/download/3.6.0.0/artifacts.tar.gz \| tar zxvf - -C /tmp/opensearch-jvector-plugin# Prepare neural-search pluginRUN mkdir -p /tmp/opensearch-neural-search && \- curl -L -s https://storage.googleapis.com/opensearch-jvector/opensearch-neural-search-3.2.0.0-20251029200300.zip \+ curl -L -s https://github.com/IBM/neural-search-jvector/releases/download/3.6.0.0/opensearch-neural-search-3.6.0.0.zip \> /tmp/opensearch-neural-search/plugin.zip# Install additional plugins-RUN opensearch-plugin install --batch file:///tmp/opensearch-jvector-plugin/repository/org/opensearch/plugin/opensearch-jvector-plugin/3.2.0.0/opensearch-jvector-plugin-3.2.0.0.zip && \+RUN opensearch-plugin install --batch file:///tmp/opensearch-jvector-plugin/repository/org/opensearch/plugin/opensearch-jvector-plugin/3.6.0.0/opensearch-jvector-plugin-3.6.0.0.zip && \opensearch-plugin install --batch file:///tmp/opensearch-neural-search/plugin.zip && \opensearch-plugin install --batch repository-gcs && \opensearch-plugin install --batch repository-azure && \# opensearch-plugin install --batch repository-s3 && \- opensearch-plugin install --batch https://github.com/opensearch-project/opensearch-prometheus-exporter/releases/download/3.2.0.0/prometheus-exporter-3.2.0.0.zip+ opensearch-plugin install --batch https://github.com/opensearch-project/opensearch-prometheus-exporter/releases/download/3.6.0.0/prometheus-exporter-3.6.0.0.zip# Apply Netty patchCOPY patch-netty.sh /tmp/@@ -67,8 +65,6 @@ RUN usermod -aG wheel opensearch && \# Copy OpenSearch from the upstream stageCOPY --from=upstream_opensearch --chown=$UID:0 $OPENSEARCH_HOME $OPENSEARCH_HOME-ARG OPENSEARCH_VERSION=3.2.0-######################################### Async-profiler (multi-arch like your original)########################################diff --git a/cloud_securityconfig/roles.yml b/cloud_securityconfig/roles.ymlindex 460e39169..1503ea891 100644--- a/cloud_securityconfig/roles.yml+++ b/cloud_securityconfig/roles.yml@@ -13,9 +13,6 @@ openrag_user_role:- index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*", "orag-*", "orag*"]allowed_actions:- read- - indices:admin/mappings/get- - indices:admin/exists- - indices:admin/getdls: >{"bool":{"should":[{"term":{"owner":"${user.name}"}},@@ -25,22 +22,31 @@ openrag_user_role:{"terms":{"allowed_principals":{"index":"openrag_dls_principals","id":"${user.name}","path":"principals"}}},{"bool":{"must_not":{"exists":{"field":"owner"}}}}],"minimum_should_match":1}}+ - index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*", "orag-*", "orag*"]+ allowed_actions:+ - indices:admin/mappings/get+ - indices:admin/exists+ - indices:admin/get- index_patterns: ["openrag_dls_principals", "openrag_dls_principals*"]allowed_actions:- read- - indices:admin/exists- indices:data/read/get- indices:data/read/searchdls: >{"term":{"user_name":"${user.name}"}}+ - index_patterns: ["openrag_dls_principals", "openrag_dls_principals*"]+ allowed_actions:+ - indices:admin/exists- index_patterns: ["api_keys", "api_keys*"]allowed_actions:- read- - indices:admin/mappings/get- - indices:admin/exists- - indices:admin/getdls: >{"bool":{"should":[{"term":{"user_id":"${user.name}"}},{"bool":{"must_not":{"exists":{"field":"user_id"}}}}],"minimum_should_match":1}}+ - index_patterns: ["api_keys", "api_keys*"]+ allowed_actions:+ - indices:admin/mappings/get+ - indices:admin/exists+ - indices:admin/getdiff --git a/docker-compose.yml b/docker-compose.ymlindex 51a9875b2..1cc1efc74 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -34,7 +34,7 @@ services:restart: unless-stoppeddashboards:- image: opensearchproject/opensearch-dashboards:3.0.0+ image: opensearchproject/opensearch-dashboards:3.6.0container_name: osdashdepends_on:- opensearchdiff --git a/kubernetes/helm/openrag/values.yaml b/kubernetes/helm/openrag/values.yamlindex 461474ae3..f7302bd72 100644--- a/kubernetes/helm/openrag/values.yaml+++ b/kubernetes/helm/openrag/values.yaml@@ -420,7 +420,7 @@ dashboards:image:repository: opensearchproject/opensearch-dashboards- tag: "3.0.0"+ tag: "3.6.0"replicaCount: 1diff --git a/patch-netty.sh b/patch-netty.shindex 2114cdcd4..d64dabed6 100644--- a/patch-netty.sh+++ b/patch-netty.sh@@ -1,7 +1,7 @@#!/bin/bashset -euo pipefail-NETTY_VERSION="4.1.125.Final"+NETTY_VERSION="4.2.15.Final"MAVEN_BASE_URL="https://repo1.maven.org/maven2/io/netty"DOWNLOAD_DIR="/tmp/netty-${NETTY_VERSION}"@@ -27,130 +27,81 @@ download_with_retry() {return 1}-# List of unique Netty artifacts-# Some of them are not used below, but we'll keep the complete set here-NETTY_ARTIFACTS=(- "netty-buffer"- "netty-codec"- "netty-codec-dns"- "netty-codec-http"- "netty-codec-http2"- "netty-codec-socks"- "netty-common"- "netty-handler"- "netty-handler-proxy"- "netty-resolver"- "netty-resolver-dns"- "netty-transport"- "netty-transport-classes-epoll"- "netty-transport-native-unix-common"+# Whitelist of core Netty artifacts to match and patch+declare -A CORE_NETTY_ARTIFACTS=(+ ["netty-buffer"]=1+ ["netty-codec"]=1+ ["netty-codec-dns"]=1+ ["netty-codec-http"]=1+ ["netty-codec-http2"]=1+ ["netty-codec-socks"]=1+ ["netty-common"]=1+ ["netty-handler"]=1+ ["netty-handler-proxy"]=1+ ["netty-resolver"]=1+ ["netty-resolver-dns"]=1+ ["netty-transport"]=1+ ["netty-transport-classes-epoll"]=1+ ["netty-transport-native-unix-common"]=1)-echo "Downloading Netty ${NETTY_VERSION} artifacts..."-for artifact in "${NETTY_ARTIFACTS[@]}"; do- jar_file="${artifact}-${NETTY_VERSION}.jar"- if [ ! -f "${DOWNLOAD_DIR}/${jar_file}" ]; then- echo " Downloading ${artifact}..."- download_with_retry "${MAVEN_BASE_URL}/${artifact}/${NETTY_VERSION}/${jar_file}" \- "${DOWNLOAD_DIR}/${jar_file}"- fi-done+replaced_count=0+matched_count=0-echo "Removing old Netty jars and replacing with ${NETTY_VERSION}..."+echo "Searching for Netty jars to patch in /usr/share/opensearch..."-# Function to replace jar with hardlink-replace_jar() {- local old_jar="$1"- local artifact_name="$2"- local new_jar="${DOWNLOAD_DIR}/${artifact_name}-${NETTY_VERSION}.jar"-- if [ -f "${old_jar}" ]; then- rm -f "${old_jar}"- # Extract directory path- local dir=$(dirname "${old_jar}")- # Create hardlink with the new version number in filename- local new_filename="${dir}/${artifact_name}-${NETTY_VERSION}.jar"- ln "${new_jar}" "${new_filename}"- echo " Replaced: ${old_jar} -> ${new_filename}"+# Find all Netty jars under /usr/share/opensearch+shopt -s globstar+for jar_path in /usr/share/opensearch/**/netty-*.jar; do+ [ -f "$jar_path" ] || continue+ filename=$(basename "$jar_path")+ dir=$(dirname "$jar_path")++ # Match netty-<artifact>-<version>.jar+ # Example: netty-buffer-4.2.12.Final.jar+ if [[ "$filename" =~ ^(netty-[a-z0-9-]+)-([0-9]+\.[0-9]+\.[0-9]+\.?[a-zA-Z0-9]*)\.jar$ ]]; then+ artifact="${BASH_REMATCH[1]}"+ version="${BASH_REMATCH[2]}"++ # Check if the artifact is one of the core Netty artifacts we want to patch+ if [[ -n "${CORE_NETTY_ARTIFACTS[$artifact]:-}" ]]; then+ matched_count=$((matched_count + 1))+ # If the version is already the target version, skip+ if [ "$version" = "$NETTY_VERSION" ]; then+ echo " Skipping: ${filename} (already version ${NETTY_VERSION})"+ continue+ fi++ echo " Found vulnerable Netty jar: ${filename} in ${dir}"++ new_jar="${DOWNLOAD_DIR}/${artifact}-${NETTY_VERSION}.jar"++ # Download the new version if it hasn't been downloaded yet+ if [ ! -f "$new_jar" ]; then+ echo " Downloading ${artifact}-${NETTY_VERSION}.jar..."+ if ! download_with_retry "${MAVEN_BASE_URL}/${artifact}/${NETTY_VERSION}/${artifact}-${NETTY_VERSION}.jar" "$new_jar"; then+ echo "ERROR: Failed to download patched dependency ${artifact}."+ exit 1+ fi+ fi++ # Replace the old jar with a hardlink to the new jar+ rm -f "$jar_path"+ new_filename="${dir}/${artifact}-${NETTY_VERSION}.jar"+ ln "$new_jar" "$new_filename"+ echo " Replaced with: ${artifact}-${NETTY_VERSION}.jar"+ replaced_count=$((replaced_count + 1))+ fifi-}--# Replace transport-netty4 module jars (4.1.121.Final -> 4.1.125.Final)-replace_jar "/usr/share/opensearch/modules/transport-netty4/netty-buffer-4.1.121.Final.jar" "netty-buffer"-replace_jar "/usr/share/opensearch/modules/transport-netty4/netty-codec-4.1.121.Final.jar" "netty-codec"-replace_jar "/usr/share/opensearch/modules/transport-netty4/netty-codec-http-4.1.121.Final.jar" "netty-codec-http"-replace_jar "/usr/share/opensearch/modules/transport-netty4/netty-codec-http2-4.1.121.Final.jar" "netty-codec-http2"-replace_jar "/usr/share/opensearch/modules/transport-netty4/netty-common-4.1.121.Final.jar" "netty-common"-replace_jar "/usr/share/opensearch/modules/transport-netty4/netty-handler-4.1.121.Final.jar" "netty-handler"-replace_jar "/usr/share/opensearch/modules/transport-netty4/netty-resolver-4.1.121.Final.jar" "netty-resolver"-replace_jar "/usr/share/opensearch/modules/transport-netty4/netty-transport-4.1.121.Final.jar" "netty-transport"-replace_jar "/usr/share/opensearch/modules/transport-netty4/netty-transport-native-unix-common-4.1.121.Final.jar" "netty-transport-native-unix-common"--# Replace opensearch-ml plugin jars (4.1.118.Final -> 4.1.125.Final)-replace_jar "/usr/share/opensearch/plugins/opensearch-ml/netty-buffer-4.1.118.Final.jar" "netty-buffer"-replace_jar "/usr/share/opensearch/plugins/opensearch-ml/netty-codec-4.1.118.Final.jar" "netty-codec"-replace_jar "/usr/share/opensearch/plugins/opensearch-ml/netty-codec-http-4.1.118.Final.jar" "netty-codec-http"-replace_jar "/usr/share/opensearch/plugins/opensearch-ml/netty-codec-http2-4.1.118.Final.jar" "netty-codec-http2"-replace_jar "/usr/share/opensearch/plugins/opensearch-ml/netty-common-4.1.118.Final.jar" "netty-common"-replace_jar "/usr/share/opensearch/plugins/opensearch-ml/netty-handler-4.1.118.Final.jar" "netty-handler"-replace_jar "/usr/share/opensearch/plugins/opensearch-ml/netty-resolver-4.1.118.Final.jar" "netty-resolver"-replace_jar "/usr/share/opensearch/plugins/opensearch-ml/netty-transport-4.1.118.Final.jar" "netty-transport"-replace_jar "/usr/share/opensearch/plugins/opensearch-ml/netty-transport-classes-epoll-4.1.118.Final.jar" "netty-transport-classes-epoll"-replace_jar "/usr/share/opensearch/plugins/opensearch-ml/netty-transport-native-unix-common-4.1.118.Final.jar" "netty-transport-native-unix-common"--# Replace opensearch-notifications plugin jars (4.1.118.Final -> 4.1.125.Final)-replace_jar "/usr/share/opensearch/plugins/opensearch-notifications/netty-buffer-4.1.118.Final.jar" "netty-buffer"-replace_jar "/usr/share/opensearch/plugins/opensearch-notifications/netty-codec-4.1.118.Final.jar" "netty-codec"-replace_jar "/usr/share/opensearch/plugins/opensearch-notifications/netty-codec-http-4.1.118.Final.jar" "netty-codec-http"-replace_jar "/usr/share/opensearch/plugins/opensearch-notifications/netty-codec-http2-4.1.118.Final.jar" "netty-codec-http2"-replace_jar "/usr/share/opensearch/plugins/opensearch-notifications/netty-common-4.1.118.Final.jar" "netty-common"-replace_jar "/usr/share/opensearch/plugins/opensearch-notifications/netty-handler-4.1.118.Final.jar" "netty-handler"-replace_jar "/usr/share/opensearch/plugins/opensearch-notifications/netty-resolver-4.1.118.Final.jar" "netty-resolver"-replace_jar "/usr/share/opensearch/plugins/opensearch-notifications/netty-transport-4.1.118.Final.jar" "netty-transport"-replace_jar "/usr/share/opensearch/plugins/opensearch-notifications/netty-transport-classes-epoll-4.1.118.Final.jar" "netty-transport-classes-epoll"-replace_jar "/usr/share/opensearch/plugins/opensearch-notifications/netty-transport-native-unix-common-4.1.118.Final.jar" "netty-transport-native-unix-common"--# Replace opensearch-performance-analyzer plugin jars (4.1.121.Final -> 4.1.125.Final)-replace_jar "/usr/share/opensearch/plugins/opensearch-performance-analyzer/netty-buffer-4.1.121.Final.jar" "netty-buffer"-replace_jar "/usr/share/opensearch/plugins/opensearch-performance-analyzer/netty-codec-4.1.121.Final.jar" "netty-codec"-replace_jar "/usr/share/opensearch/plugins/opensearch-performance-analyzer/netty-codec-http-4.1.121.Final.jar" "netty-codec-http"-replace_jar "/usr/share/opensearch/plugins/opensearch-performance-analyzer/netty-codec-http2-4.1.121.Final.jar" "netty-codec-http2"-replace_jar "/usr/share/opensearch/plugins/opensearch-performance-analyzer/netty-codec-socks-4.1.121.Final.jar" "netty-codec-socks"-replace_jar "/usr/share/opensearch/plugins/opensearch-performance-analyzer/netty-common-4.1.121.Final.jar" "netty-common"-replace_jar "/usr/share/opensearch/plugins/opensearch-performance-analyzer/netty-handler-4.1.121.Final.jar" "netty-handler"-replace_jar "/usr/share/opensearch/plugins/opensearch-performance-analyzer/netty-handler-proxy-4.1.121.Final.jar" "netty-handler-proxy"-replace_jar "/usr/share/opensearch/plugins/opensearch-performance-analyzer/netty-resolver-4.1.121.Final.jar" "netty-resolver"-replace_jar "/usr/share/opensearch/plugins/opensearch-performance-analyzer/netty-transport-4.1.121.Final.jar" "netty-transport"-replace_jar "/usr/share/opensearch/plugins/opensearch-performance-analyzer/netty-transport-native-unix-common-4.1.121.Final.jar" "netty-transport-native-unix-common"--# Replace opensearch-security plugin jars (4.1.121.Final -> 4.1.125.Final)-replace_jar "/usr/share/opensearch/plugins/opensearch-security/netty-buffer-4.1.121.Final.jar" "netty-buffer"-replace_jar "/usr/share/opensearch/plugins/opensearch-security/netty-codec-4.1.121.Final.jar" "netty-codec"-replace_jar "/usr/share/opensearch/plugins/opensearch-security/netty-codec-http-4.1.121.Final.jar" "netty-codec-http"-replace_jar "/usr/share/opensearch/plugins/opensearch-security/netty-codec-http2-4.1.121.Final.jar" "netty-codec-http2"-replace_jar "/usr/share/opensearch/plugins/opensearch-security/netty-common-4.1.121.Final.jar" "netty-common"-replace_jar "/usr/share/opensearch/plugins/opensearch-security/netty-handler-4.1.121.Final.jar" "netty-handler"-replace_jar "/usr/share/opensearch/plugins/opensearch-security/netty-resolver-4.1.121.Final.jar" "netty-resolver"-replace_jar "/usr/share/opensearch/plugins/opensearch-security/netty-transport-4.1.121.Final.jar" "netty-transport"-replace_jar "/usr/share/opensearch/plugins/opensearch-security/netty-transport-native-unix-common-4.1.121.Final.jar" "netty-transport-native-unix-common"--# Replace repository-azure plugin jars (4.1.121.Final -> 4.1.125.Final)-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-buffer-4.1.121.Final.jar" "netty-buffer"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-codec-4.1.121.Final.jar" "netty-codec"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-codec-dns-4.1.121.Final.jar" "netty-codec-dns"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-codec-http-4.1.121.Final.jar" "netty-codec-http"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-codec-http2-4.1.121.Final.jar" "netty-codec-http2"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-codec-socks-4.1.121.Final.jar" "netty-codec-socks"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-common-4.1.121.Final.jar" "netty-common"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-handler-4.1.121.Final.jar" "netty-handler"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-handler-proxy-4.1.121.Final.jar" "netty-handler-proxy"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-resolver-4.1.121.Final.jar" "netty-resolver"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-resolver-dns-4.1.121.Final.jar" "netty-resolver-dns"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-transport-4.1.121.Final.jar" "netty-transport"-replace_jar "/usr/share/opensearch/plugins/repository-azure/netty-transport-native-unix-common-4.1.121.Final.jar" "netty-transport-native-unix-common"+done-# Remove the download directory after hardlinking+# Clean up download directoryrm -rf "${DOWNLOAD_DIR}"-echo "Successfully replaced all old Netty jars with ${NETTY_VERSION}"-echo "Hardlinks used to minimize disk space"+echo "Netty patching complete. Replaced ${replaced_count} jars."++# Fail-safe check: If no Netty jars were matched at all, we must fail the build.+if [ "${matched_count}" -eq 0 ]; then+ echo "ERROR: No Netty jars were found to be patched! This indicates that the script failed to find any expected jars."+ exit 1+fidiff --git a/securityconfig/roles.yml b/securityconfig/roles.ymlindex 45b45405a..d19c76dc1 100644--- a/securityconfig/roles.yml+++ b/securityconfig/roles.yml@@ -13,9 +13,6 @@ openrag_user_role:- index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*"]allowed_actions:- read- - indices:admin/mappings/get- - indices:admin/exists- - indices:admin/getdls: >{"bool":{"should":[{"term":{"owner":"${user.name}"}},@@ -25,22 +22,31 @@ openrag_user_role:{"terms":{"allowed_principals":{"index":"openrag_dls_principals","id":"${user.name}","path":"principals"}}},{"bool":{"must_not":{"exists":{"field":"owner"}}}}],"minimum_should_match":1}}+ - index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*"]+ allowed_actions:+ - indices:admin/mappings/get+ - indices:admin/exists+ - indices:admin/get- index_patterns: ["openrag_dls_principals", "openrag_dls_principals*"]allowed_actions:- read- - indices:admin/exists- indices:data/read/get- indices:data/read/searchdls: >{"term":{"user_name":"${user.name}"}}+ - index_patterns: ["openrag_dls_principals", "openrag_dls_principals*"]+ allowed_actions:+ - indices:admin/exists- index_patterns: ["api_keys", "api_keys*"]allowed_actions:- read- - indices:admin/mappings/get- - indices:admin/exists- - indices:admin/getdls: >{"bool":{"should":[{"term":{"user_id":"${user.name}"}},{"bool":{"must_not":{"exists":{"field":"user_id"}}}}],"minimum_should_match":1}}+ - index_patterns: ["api_keys", "api_keys*"]+ allowed_actions:+ - indices:admin/mappings/get+ - indices:admin/exists+ - indices:admin/getdiff --git a/tests/unit/test_group_acl.py b/tests/unit/test_group_acl.pyindex 932f54eb8..3af6235ae 100644--- a/tests/unit/test_group_acl.py+++ b/tests/unit/test_group_acl.py@@ -510,8 +510,8 @@ def test_dls_principal_service_uses_basic_admin_client_in_ibm(monkeypatch):clients = Clients()monkeypatch.setattr("config.settings.IBM_AUTH_ENABLED", True)- monkeypatch.setattr("config.settings.OPENSEARCH_USERNAME", "admin-user")- monkeypatch.setattr("config.settings.OPENSEARCH_PASSWORD", "admin-pass")+ monkeypatch.setenv("OPENSEARCH_USERNAME", "admin-user")+ monkeypatch.setenv("OPENSEARCH_PASSWORD", "admin-pass")monkeypatch.setattr("config.settings.clients", clients)service = DLSPrincipalService(connector_service=None)