OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
chore: merge Saas features to main (#1359)
Edwin Jose8571cfa9b3ae
diff --git a/.gitignore b/.gitignoreindex 8ee2c4059..5391089d7 100644--- a/.gitignore+++ b/.gitignore@@ -49,3 +49,4 @@ documents/openrag-documentation.pdfdocuments/ibm_anthropic.pdfdocuments/docling.pdf/opensearch-data-new-lf+/opensearch-data2diff --git a/Dockerfile b/Dockerfileindex e4e532514..658c092fa 100644--- a/Dockerfile+++ b/Dockerfile@@ -99,7 +99,8 @@ RUN echo "#!/bin/bash" > /usr/share/opensearch/profile.sh && \# Copy OIDC and DLS security configuration (as root, like before)COPY securityconfig/ /usr/share/opensearch/securityconfig/-RUN chown -R opensearch:opensearch /usr/share/opensearch/securityconfig/+COPY cloud_securityconfig/ /usr/share/opensearch/cloud_securityconfig/+RUN chown -R opensearch:opensearch /usr/share/opensearch/securityconfig/ /usr/share/opensearch/cloud_securityconfig/# Create a script to apply security configuration after OpenSearch startsRUN echo '#!/bin/bash' > /usr/share/opensearch/setup-security.sh && \diff --git a/Dockerfile.backend b/Dockerfile.backendindex 5fd4226ae..c42b4053e 100644--- a/Dockerfile.backend+++ b/Dockerfile.backend@@ -39,6 +39,8 @@ RUN --mount=type=cache,target=/root/.cache/uv,uid=0,gid=0 \# Application (invalidates after deps only when src/flows change)COPY src/ ./src/COPY flows/ ./flows/+COPY securityconfig/ ./securityconfig/+COPY cloud_securityconfig/ ./cloud_securityconfig/# -----------------------------------------------------------------------------# Stage: runtime (minimal image)@@ -54,6 +56,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \WORKDIR /appCOPY --from=builder /app /app+COPY securityconfig/ ./securityconfig/+COPY cloud_securityconfig/ ./cloud_securityconfig/ENV VIRTUAL_ENV=/app/.venvENV PATH="/app/.venv/bin:$PATH"diff --git a/cloud_securityconfig/action_groups.yml b/cloud_securityconfig/action_groups.ymlnew file mode 100644index 000000000..7886266f4--- /dev/null+++ b/cloud_securityconfig/action_groups.yml@@ -0,0 +1,5 @@+_meta:+ type: "actiongroups"+ config_version: 2++# Empty - using built-in OpenSearch action groups only\ No newline at end of filediff --git a/cloud_securityconfig/config.yml b/cloud_securityconfig/config.ymlnew file mode 100644index 000000000..8f836da59--- /dev/null+++ b/cloud_securityconfig/config.yml@@ -0,0 +1,29 @@+_meta:+ type: "config"+ config_version: 2++config:+ dynamic:+ authc:+ openid_auth_domain:+ order: 0+ http_enabled: true+ http_authenticator:+ type: openid+ challenge: false+ config:+ openid_connect_url: "http://openrag-backend:8000/.well-known/openid-configuration"+ subject_key: "sub"+ jwt_header: "Authorization" # expects Bearer token+ roles_key: "roles"+ authentication_backend:+ type: noop++ basic_internal_auth_domain:+ order: 1+ http_enabled: true+ http_authenticator:+ type: basic+ challenge: true+ authentication_backend:+ type: interndiff --git a/cloud_securityconfig/internal_users.yml b/cloud_securityconfig/internal_users.ymlnew file mode 100644index 000000000..f467581be--- /dev/null+++ b/cloud_securityconfig/internal_users.yml@@ -0,0 +1,7 @@+_meta:+ type: "internalusers"+ config_version: 2++admin:+ reserved: true+ hash: "ADMIN_HASH_PLACEHOLDER"diff --git a/cloud_securityconfig/nodes_dn.yml b/cloud_securityconfig/nodes_dn.ymlnew file mode 100644index 000000000..8ce6b65ed--- /dev/null+++ b/cloud_securityconfig/nodes_dn.yml@@ -0,0 +1,5 @@+_meta:+ type: "nodesdn"+ config_version: 2++# Empty nodes DN - using default node certificate validation\ No newline at end of filediff --git a/cloud_securityconfig/roles.yml b/cloud_securityconfig/roles.ymlnew file mode 100644index 000000000..7f38a6a7c--- /dev/null+++ b/cloud_securityconfig/roles.yml@@ -0,0 +1,57 @@+_meta:+ type: "roles"+ config_version: 2++openrag_user_role:+ description: "DLS: user can read/write docs they own or are allowed on"+ cluster_permissions:+ - "indices:data/write/bulk"+ - "indices:data/write/index"+ - "indices:data/read/scroll"+ - "indices:data/read/scroll/clear"+ - "cluster:monitor/*"+ - "cluster:admin/opensearch/notifications/configs/create"+ - "cluster:admin/opensearch/notifications/configs/list"+ - "cluster:admin/opensearch/notifications/configs/get"+ - "cluster:admin/opensearch/notifications/configs/update"+ - "cluster:admin/opensearch/notifications/configs/delete"+ - "cluster:admin/opensearch/alerting/*"++ index_permissions:+ - index_patterns: ["documents", "documents*", "knowledge_filters", "knowledge_filters*", "orag-*",orag*]+ allowed_actions:+ - crud+ - create_index+ - indices:admin/mappings/get+ - indices:admin/mappings/put+ - indices:admin/exists+ - indices:admin/get+ - indices:admin/refresh+ - indices:data/write/delete/byquery+ - indices:data/write/update/byquery+ dls: >+ {"bool":{"should":[+ {"term":{"owner":"${user.name}"}},+ {"term":{"allowed_users":"${user.name}"}},+ {"bool":{"must_not":{"exists":{"field":"owner"}}}}+ ],"minimum_should_match":1}}+ - index_patterns: ["api_keys", "api_keys*"]+ allowed_actions:+ - crud+ - create_index+ - indices:admin/mappings/get+ - indices:admin/mappings/put+ - indices:admin/exists+ - indices:admin/get+ - indices:admin/refresh+ - indices:data/write/delete/byquery+ - indices:data/write/update/byquery+ dls: >+ {"bool":{"should":[+ {"term":{"user_id":"${user.name}"}},+ {"bool":{"must_not":{"exists":{"field":"user_id"}}}}+ ],"minimum_should_match":1}}+ - index_patterns: [".opendistro-alerting-config"]+ allowed_actions:+ - crud+ - indices:admin/getdiff --git a/cloud_securityconfig/roles_mapping.yml b/cloud_securityconfig/roles_mapping.ymlnew file mode 100644index 000000000..8329b5911--- /dev/null+++ b/cloud_securityconfig/roles_mapping.yml@@ -0,0 +1,16 @@+_meta:+ type: "rolesmapping"+ config_version: 2++openrag_user_role:+ users: []+ hosts: []+ backend_roles:+ - "openrag_user"+ - "all_access"++all_access:+ users:+ - "admin"+ backend_roles:+ - "admin"diff --git a/cloud_securityconfig/tenants.yml b/cloud_securityconfig/tenants.ymlnew file mode 100644index 000000000..01b8c6ffe--- /dev/null+++ b/cloud_securityconfig/tenants.yml@@ -0,0 +1,5 @@+_meta:+ type: "tenants"+ config_version: 2++# Empty tenants - using global tenant only\ No newline at end of filediff --git a/docker-compose.yml b/docker-compose.ymlindex 8a1d31ad7..b1a5c175d 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -1,15 +1,30 @@services:opensearch:image: langflowai/openrag-opensearch:${OPENRAG_VERSION:-latest}- #build:- #context: .- #dockerfile: Dockerfile+ # build:+ # context: .+ # dockerfile: Dockerfilecontainer_name: osdepends_on:- openrag-backendenvironment:- discovery.type=single-node- OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}+ # Run security setup in background after OpenSearch starts+ command: >+ bash -c "+ # Ensure data directory has correct permissions+ sudo chown -R opensearch:opensearch /usr/share/opensearch/data || true++ # Start OpenSearch in background+ /usr/share/opensearch/opensearch-docker-entrypoint.sh opensearch &++ # Wait a bit for OpenSearch to start, then apply security config+ sleep 10 && /usr/share/opensearch/setup-security.sh &++ # Wait for background processes+ wait+ "- OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1gports:- "9200:9200"diff --git a/frontend/components/knowledge-search-bar.tsx b/frontend/components/knowledge-search-bar.tsxindex 89216ad0b..03a449173 100644--- a/frontend/components/knowledge-search-bar.tsx+++ b/frontend/components/knowledge-search-bar.tsx@@ -47,31 +47,30 @@ export const KnowledgeSearchBar = () => {const refreshOpenragDocsMutation = useRefreshOpenragDocs();return (- <form- onSubmit={handleSearch}- className={"relative flex w-full items-stretch"}- >- {!!selectedFilter?.name && (- <div- title={selectedFilter.name}- className={cn(- "absolute -top-3 left-0 z-10 flex h-8 max-w-[260px] items-center gap-2 rounded-md border border-primary/60 px-3 text-sm shadow-sm",- filterAccentClasses[parsedFilterData?.color || "zinc"],- )}- >- <span className="truncate font-medium">{selectedFilter.name}</span>- <button- type="button"- aria-label="Remove filter"- className="inline-flex h-4 w-4 flex-shrink-0 items-center justify-center opacity-80 transition-opacity hover:opacity-100"- onClick={() => setSelectedFilter(null)}+ <form onSubmit={handleSearch} className={"flex w-full items-stretch"}>+ <div className="flex h-12 w-full overflow-hidden border border-border bg-card">+ {!!selectedFilter?.name && (+ <div+ title={selectedFilter.name}+ className={cn(+ "flex h-full flex-shrink-0 items-center gap-1.5 border-r border-border px-2 max-w-[200px]",+ filterAccentClasses[parsedFilterData?.color || "zinc"],+ )}>- <X className="h-3.5 w-3.5" />- </button>- </div>- )}+ <span className="truncate text-xs font-medium">+ {selectedFilter.name}+ </span>+ <button+ type="button"+ aria-label="Remove filter"+ className="inline-flex h-4 w-4 flex-shrink-0 items-center justify-center opacity-80 transition-opacity hover:opacity-100"+ onClick={() => setSelectedFilter(null)}+ >+ <X className="h-3.5 w-3.5" />+ </button>+ </div>+ )}- <div className="flex h-12 w-full overflow-hidden border border-border bg-card"><div className="flex h-full flex-shrink-0 items-center justify-center"><SearchclassName="h-4 w-4 m-4 text-[var(--icon-secondary)]"diff --git a/pyproject.toml b/pyproject.tomlindex 047c16099..83dd44f36 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -45,6 +45,7 @@ dependencies = ["openai>=1.0.0","pyyaml>=6.0","tiktoken>=0.7.0",+ "prometheus-fastapi-instrumentator>=7.1.0",][dependency-groups]diff --git a/scripts/test.sh b/scripts/test.shnew file mode 100755index 000000000..622a34fb3--- /dev/null+++ b/scripts/test.sh@@ -0,0 +1,367 @@+#!/usr/bin/env bash+set -euo pipefail++# ═══════════════════════════════════════════════════════════+# OpenSearch ACL Test Script+# Tests Document-Level Security (DLS) by searching existing documents+# ═══════════════════════════════════════════════════════════++OS_URL="${OS_URL:?Set OS_URL (e.g. https://host:9200)}"+USER_A="${USER_A:?Set USER_A (user:password for first user)}"+USER_B="${USER_B:?Set USER_B (user:password for second user)}"+INDEX="${INDEX:-documents}"++# Actual user identity used as document owner (may differ from API key username)+USER_A_NAME="${USER_A_OWNER:?Set USER_A_OWNER}"+USER_B_NAME="${USER_B_OWNER:?Set USER_B_OWNER}"++PASS=0+FAIL=0++green() { printf "\033[32m%s\033[0m\n" "$*"; }+red() { printf "\033[31m%s\033[0m\n" "$*"; }+yellow() { printf "\033[33m%s\033[0m\n" "$*"; }+bold() { printf "\033[1m%s\033[0m\n" "$*"; }+dim() { printf "\033[2m%s\033[0m\n" "$*"; }++check() {+ local label="$1" expected="$2" actual="$3"+ if [ "$actual" = "$expected" ]; then+ green " ✓ $label (expected=$expected, got=$actual)"+ PASS=$((PASS + 1))+ else+ red " ✗ $label (expected=$expected, got=$actual)"+ FAIL=$((FAIL + 1))+ fi+}++curl_os() {+ local creds="$1"; shift+ curl -sk -u "$creds" "$@"+}++# ───────────────────────────────────────────────+bold "Step 0: Cluster health"+# ───────────────────────────────────────────────++HEALTH=$(curl_os "$USER_A" "$OS_URL/_cluster/health" 2>/dev/null)+echo "$HEALTH" | python3 -c "+import sys, json+h = json.load(sys.stdin)+status = h.get('status', 'unknown')+color = {'green': '\033[32m', 'yellow': '\033[33m', 'red': '\033[31m'}.get(status, '')+reset = '\033[0m'+print(f' Status: {color}{status}{reset}')+print(f' Cluster name: {h.get(\"cluster_name\", \"?\")}')+print(f' Nodes: {h.get(\"number_of_nodes\", \"?\")}')+print(f' Data nodes: {h.get(\"number_of_data_nodes\", \"?\")}')+print(f' Active shards: {h.get(\"active_shards\", \"?\")}')+print(f' Relocating shards: {h.get(\"relocating_shards\", \"?\")}')+print(f' Unassigned shards: {h.get(\"unassigned_shards\", \"?\")}')+print(f' Pending tasks: {h.get(\"number_of_pending_tasks\", \"?\")}')+" 2>/dev/null || red " Could not fetch cluster health"+echo ""++# ───────────────────────────────────────────────+bold "Step 1: Verify authentication"+# ───────────────────────────────────────────────++echo "User A ($USER_A_NAME):"+ACCT_A=$(curl_os "$USER_A" "$OS_URL/_plugins/_security/api/account" 2>/dev/null)+ROLES_A=$(echo "$ACCT_A" | python3 -c "import sys,json; print(','.join(json.load(sys.stdin).get('roles',[])))" 2>/dev/null || echo "PARSE_ERROR")+BROLES_A=$(echo "$ACCT_A" | python3 -c "import sys,json; print(','.join(json.load(sys.stdin).get('backend_roles',[])))" 2>/dev/null || echo "PARSE_ERROR")+echo " roles: $ROLES_A"+echo " backend_roles: $BROLES_A"+if [ "$ROLES_A" = "PARSE_ERROR" ]; then+ red " ERROR: could not parse account info. Raw response:"+ dim " $ACCT_A"+fi++echo "User B ($USER_B_NAME):"+ACCT_B=$(curl_os "$USER_B" "$OS_URL/_plugins/_security/api/account" 2>/dev/null)+ROLES_B=$(echo "$ACCT_B" | python3 -c "import sys,json; print(','.join(json.load(sys.stdin).get('roles',[])))" 2>/dev/null || echo "PARSE_ERROR")+BROLES_B=$(echo "$ACCT_B" | python3 -c "import sys,json; print(','.join(json.load(sys.stdin).get('backend_roles',[])))" 2>/dev/null || echo "PARSE_ERROR")+echo " roles: $ROLES_B"+echo " backend_roles: $BROLES_B"+if [ "$ROLES_B" = "PARSE_ERROR" ]; then+ red " ERROR: could not parse account info. Raw response:"+ dim " $ACCT_B"+fi++if echo "$ROLES_A" | grep -q "all_access"; then+ yellow " ⚠ User A has all_access — OpenSearch DLS will be bypassed"+fi+if echo "$ROLES_B" | grep -q "all_access"; then+ yellow " ⚠ User B has all_access — OpenSearch DLS will be bypassed"+fi+echo ""++# ───────────────────────────────────────────────+bold "Step 2: List indices & check mapping"+# ───────────────────────────────────────────────++echo " All indices on cluster:"+curl_os "$USER_A" "$OS_URL/_cat/indices?format=json" 2>/dev/null \+ | python3 -c "+import sys, json+indices = json.load(sys.stdin)+indices.sort(key=lambda x: int(x.get('docs.count') or 0), reverse=True)+for idx in indices:+ name = idx.get('index', '?')+ cnt = idx.get('docs.count', '0') or '0'+ sz = idx.get('store.size', '?') or '?'+ if int(cnt) > 0:+ print(f' \033[32m{name:<40} docs={cnt} size={sz}\033[0m')+ else:+ print(f' {name:<40} docs={cnt} size={sz}')+" 2>/dev/null+echo ""++INDEX_STATUS=$(curl_os "$USER_A" -o /dev/null -w "%{http_code}" "$OS_URL/$INDEX" 2>/dev/null || echo "000")+if [ "$INDEX_STATUS" = "200" ]; then+ green " Index '$INDEX' exists"+else+ red " Index '$INDEX' does NOT exist (HTTP $INDEX_STATUS)"+ echo " Available indices:"+ curl_os "$USER_A" "$OS_URL/_cat/indices?v&h=index,docs.count,store.size" 2>/dev/null | while IFS= read -r line; do echo " $line"; done+ exit 1+fi++MAPPING_RAW=$(curl_os "$USER_A" "$OS_URL/$INDEX/_mapping" 2>/dev/null)+OWNER_TYPE=$(echo "$MAPPING_RAW" | python3 -c "+import sys, json+m = json.load(sys.stdin)+for idx in m.values():+ props = idx.get('mappings',{}).get('properties',{})+ print(props.get('owner',{}).get('type','MISSING'))+ break+" 2>/dev/null || echo "PARSE_ERROR")+ALLOWED_TYPE=$(echo "$MAPPING_RAW" | python3 -c "+import sys, json+m = json.load(sys.stdin)+for idx in m.values():+ props = idx.get('mappings',{}).get('properties',{})+ print(props.get('allowed_users',{}).get('type','MISSING'))+ break+" 2>/dev/null || echo "PARSE_ERROR")++check "owner field type is keyword" "keyword" "$OWNER_TYPE"+check "allowed_users field type is keyword" "keyword" "$ALLOWED_TYPE"+echo ""++parse_search_count() {+ python3 -c "import sys,json; print(json.load(sys.stdin).get('hits',{}).get('total',{}).get('value',-1))" 2>/dev/null || echo "-1"+}++# ───────────────────────────────────────────────+bold "Step 3: Document overview"+# ───────────────────────────────────────────────++OVERVIEW=$(curl_os "$USER_A" -H "Content-Type: application/json" \+ -X POST "$OS_URL/$INDEX/_search" \+ -d '{+ "size": 0,+ "query": {"match_all": {}},+ "aggs": {+ "owners": {"terms": {"field": "owner", "size": 50}},+ "no_owner": {"missing": {"field": "owner"}}+ }+ }' 2>/dev/null)++TOTAL=$(echo "$OVERVIEW" | parse_search_count)+echo " Total documents in index: $TOTAL"++echo " Documents by owner:"+echo "$OVERVIEW" | python3 -c "+import sys, json+data = json.load(sys.stdin)+aggs = data.get('aggregations', {})+buckets = aggs.get('owners', {}).get('buckets', [])+for b in buckets:+ print(f' {b[\"key\"]}: {b[\"doc_count\"]} docs')+no_owner = aggs.get('no_owner', {}).get('doc_count', 0)+if no_owner:+ print(f' (no owner): {no_owner} docs')+" 2>/dev/null || yellow " Could not parse owner aggregation"+echo ""++# ───────────────────────────────────────────────+bold "Step 4: Search as each user (DLS test)"+# ───────────────────────────────────────────────++echo "User A ($USER_A_NAME) — unfiltered search:"+RESULT_A=$(curl_os "$USER_A" -H "Content-Type: application/json" \+ -X POST "$OS_URL/$INDEX/_search" \+ -d '{"size":0,"query":{"match_all":{}}}' 2>/dev/null)+COUNT_A=$(echo "$RESULT_A" | parse_search_count)+echo " Docs visible: $COUNT_A"+if [ "$COUNT_A" = "-1" ]; then+ yellow " Debug — raw response:"+ dim " $(echo "$RESULT_A" | head -c 500)"+fi++echo "User B ($USER_B_NAME) — unfiltered search:"+RESULT_B=$(curl_os "$USER_B" -H "Content-Type: application/json" \+ -X POST "$OS_URL/$INDEX/_search" \+ -d '{"size":0,"query":{"match_all":{}}}' 2>/dev/null)+COUNT_B=$(echo "$RESULT_B" | parse_search_count)+echo " Docs visible: $COUNT_B"+if [ "$COUNT_B" = "-1" ]; then+ yellow " Debug — raw response:"+ dim " $(echo "$RESULT_B" | head -c 500)"+fi++if [ "$COUNT_A" = "$COUNT_B" ] && [ "$COUNT_A" = "$TOTAL" ]; then+ yellow " ⚠ Both users see ALL $TOTAL docs — DLS is NOT filtering (likely all_access)"+elif [ "$COUNT_A" != "$COUNT_B" ]; then+ green " ✓ Users see different doc counts — DLS appears to be working"+fi+echo ""++echo "User A — owners visible:"+curl_os "$USER_A" -H "Content-Type: application/json" \+ -X POST "$OS_URL/$INDEX/_search" \+ -d '{"size":0,"query":{"match_all":{}},"aggs":{"owners":{"terms":{"field":"owner","size":50}},"no_owner":{"missing":{"field":"owner"}}}}' 2>/dev/null \+ | python3 -c "+import sys, json+data = json.load(sys.stdin)+aggs = data.get('aggregations', {})+for b in aggs.get('owners', {}).get('buckets', []):+ print(f' {b[\"key\"]}: {b[\"doc_count\"]}')+no_owner = aggs.get('no_owner', {}).get('doc_count', 0)+if no_owner:+ print(f' (no owner): {no_owner}')+" 2>/dev/null || yellow " Could not parse"++echo "User B — owners visible:"+curl_os "$USER_B" -H "Content-Type: application/json" \+ -X POST "$OS_URL/$INDEX/_search" \+ -d '{"size":0,"query":{"match_all":{}},"aggs":{"owners":{"terms":{"field":"owner","size":50}},"no_owner":{"missing":{"field":"owner"}}}}' 2>/dev/null \+ | python3 -c "+import sys, json+data = json.load(sys.stdin)+aggs = data.get('aggregations', {})+for b in aggs.get('owners', {}).get('buckets', []):+ print(f' {b[\"key\"]}: {b[\"doc_count\"]}')+no_owner = aggs.get('no_owner', {}).get('doc_count', 0)+if no_owner:+ print(f' (no owner): {no_owner}')+" 2>/dev/null || yellow " Could not parse"+echo ""++# ───────────────────────────────────────────────+bold "Step 5: Application-level ACL filter"+# ───────────────────────────────────────────────++echo "User A ($USER_A_NAME) — with ACL filter:"+APP_RESULT_A=$(curl_os "$USER_A" -H "Content-Type: application/json" \+ -X POST "$OS_URL/$INDEX/_search" \+ -d "{+ \"size\":0,+ \"query\":{+ \"bool\":{+ \"should\":[+ {\"term\":{\"owner\":\"$USER_A_NAME\"}},+ {\"term\":{\"allowed_users\":\"$USER_A_NAME\"}},+ {\"bool\":{\"must_not\":{\"exists\":{\"field\":\"owner\"}}}}+ ],+ \"minimum_should_match\":1+ }+ }+ }" 2>/dev/null)+APP_COUNT_A=$(echo "$APP_RESULT_A" | parse_search_count)+echo " Docs visible: $APP_COUNT_A"+if [ "$APP_COUNT_A" = "-1" ]; then+ yellow " Debug — raw response:"+ dim " $(echo "$APP_RESULT_A" | head -c 500)"+fi++echo "User B ($USER_B_NAME) — with ACL filter:"+APP_RESULT_B=$(curl_os "$USER_B" -H "Content-Type: application/json" \+ -X POST "$OS_URL/$INDEX/_search" \+ -d "{+ \"size\":0,+ \"query\":{+ \"bool\":{+ \"should\":[+ {\"term\":{\"owner\":\"$USER_B_NAME\"}},+ {\"term\":{\"allowed_users\":\"$USER_B_NAME\"}},+ {\"bool\":{\"must_not\":{\"exists\":{\"field\":\"owner\"}}}}+ ],+ \"minimum_should_match\":1+ }+ }+ }" 2>/dev/null)+APP_COUNT_B=$(echo "$APP_RESULT_B" | parse_search_count)+echo " Docs visible: $APP_COUNT_B"+if [ "$APP_COUNT_B" = "-1" ]; then+ yellow " Debug — raw response:"+ dim " $(echo "$APP_RESULT_B" | head -c 500)"+fi++echo ""+if [ "$APP_COUNT_A" != "$APP_COUNT_B" ]; then+ green " ✓ App-level filter: users see different counts (A=$APP_COUNT_A, B=$APP_COUNT_B)"+ check "App filter reduces User A's visible docs" "true" \+ "$([ "$APP_COUNT_A" -lt "$COUNT_A" ] 2>/dev/null && echo true || echo false)"+ check "App filter reduces User B's visible docs" "true" \+ "$([ "$APP_COUNT_B" -lt "$COUNT_B" ] 2>/dev/null && echo true || echo false)"+elif [ "$APP_COUNT_A" = "0" ] && [ "$APP_COUNT_B" = "0" ]; then+ red " ✗ Both users see 0 docs — owner field may not match usernames"+ echo ""+ yellow " Owner values in index vs. usernames used in filter:"+ yellow " User A owner: $USER_A_NAME"+ yellow " User B owner: $USER_B_NAME"+ yellow " Owner values in index: (see Step 3 above)"+ yellow " If these don't match, the term query won't find anything."+else+ yellow " ⚠ Both users see same count ($APP_COUNT_A) — check owner field values"+fi+echo ""++# ───────────────────────────────────────────────+bold "Step 6: Sample documents (first 3)"+# ───────────────────────────────────────────────++echo "User A — sample docs:"+curl_os "$USER_A" -H "Content-Type: application/json" \+ -X POST "$OS_URL/$INDEX/_search" \+ -d '{"query":{"match_all":{}},"_source":["document_id","filename","owner","allowed_users"],"size":3}' 2>/dev/null \+ | python3 -c "+import sys, json+data = json.load(sys.stdin)+for h in data.get('hits',{}).get('hits',[]):+ s = h.get('_source',{})+ print(f' id={h[\"_id\"][:30]} owner={s.get(\"owner\",\"NONE\")} allowed={s.get(\"allowed_users\",[])} file={s.get(\"filename\",\"?\")}')+" 2>/dev/null || yellow " Could not parse"++echo "User B — sample docs:"+curl_os "$USER_B" -H "Content-Type: application/json" \+ -X POST "$OS_URL/$INDEX/_search" \+ -d '{"query":{"match_all":{}},"_source":["document_id","filename","owner","allowed_users"],"size":3}' 2>/dev/null \+ | python3 -c "+import sys, json+data = json.load(sys.stdin)+for h in data.get('hits',{}).get('hits',[]):+ s = h.get('_source',{})+ print(f' id={h[\"_id\"][:30]} owner={s.get(\"owner\",\"NONE\")} allowed={s.get(\"allowed_users\",[])} file={s.get(\"filename\",\"?\")}')+" 2>/dev/null || yellow " Could not parse"+echo ""++# ───────────────────────────────────────────────+bold "═══ Results ═══"+# ───────────────────────────────────────────────+echo " Total in index: $TOTAL"+echo " User A unfiltered: $COUNT_A"+echo " User B unfiltered: $COUNT_B"+echo " User A with app ACL filter: $APP_COUNT_A"+echo " User B with app ACL filter: $APP_COUNT_B"+echo ""+green " Passed: $PASS"+if [ "$FAIL" -gt 0 ]; then+ red " Failed: $FAIL"+ exit 1+else+ echo " Failed: 0"+ green " All checks passed!"+fidiff --git a/securityconfig/roles.yml b/securityconfig/roles.ymlindex 163bcb4c5..ba72f75e0 100644--- a/securityconfig/roles.yml+++ b/securityconfig/roles.yml@@ -9,6 +9,7 @@ openrag_user_role:- "indices:data/write/index"- "indices:data/read/scroll"- "indices:data/read/scroll/clear"+ - "cluster:monitor/*"- "cluster:admin/opensearch/notifications/configs/create"- "cluster:admin/opensearch/notifications/configs/list"- "cluster:admin/opensearch/notifications/configs/get"@@ -25,9 +26,32 @@ openrag_user_role:- indices:admin/mappings/put- indices:admin/exists- indices:admin/get+ - indices:admin/refresh+ - indices:data/write/delete/byquery+ - indices:data/write/update/byquerydls: >{"bool":{"should":[{"term":{"owner":"${user.name}"}},{"term":{"allowed_users":"${user.name}"}},{"bool":{"must_not":{"exists":{"field":"owner"}}}}],"minimum_should_match":1}}+ - index_patterns: ["api_keys", "api_keys*"]+ allowed_actions:+ - crud+ - create_index+ - indices:admin/mappings/get+ - indices:admin/mappings/put+ - indices:admin/exists+ - indices:admin/get+ - indices:admin/refresh+ - indices:data/write/delete/byquery+ - indices:data/write/update/byquery+ dls: >+ {"bool":{"should":[+ {"term":{"user_id":"${user.name}"}},+ {"bool":{"must_not":{"exists":{"field":"user_id"}}}}+ ],"minimum_should_match":1}}+ - index_patterns: [".opendistro-alerting-config"]+ allowed_actions:+ - crud+ - indices:admin/getdiff --git a/src/api/settings.py b/src/api/settings.pyindex f4dd8a3dd..97e8a8931 100644--- a/src/api/settings.py+++ b/src/api/settings.py@@ -1125,7 +1125,7 @@ async def onboarding(# Initialize the OpenSearch index if embedding model is configuredif body.embedding_model or body.embedding_provider:try:- from main import init_index_when_ready+ from main import init_indexfrom config.settings import IBM_AUTH_ENABLED, clients as app_clientsopensearch_client = None@@ -1135,7 +1135,8 @@ async def onboarding(logger.info("Initializing OpenSearch index after onboarding configuration")- await init_index_when_ready(opensearch_client)+ admin_username = user.user_id if IBM_AUTH_ENABLED and user else None+ await init_index(opensearch_client, admin_username=admin_username)logger.info("OpenSearch index initialization completed successfully")except Exception as e:logger.error(@@ -1671,8 +1672,6 @@ async def rollback_onboarding({"error": "No onboarding configuration to rollback"}, status_code=400)- jwt_token = user.jwt_token-logger.info("Rolling back onboarding configuration due to file failures")# Get all tasks for the user@@ -1735,7 +1734,7 @@ async def rollback_onboarding(if filename:try:opensearch_client = session_manager.get_user_opensearch_client(- user.user_id, jwt_token+ user.user_id, user.jwt_token)from utils.opensearch_queries import build_filename_delete_bodyfrom config.settings import get_index_namediff --git a/src/dependencies.py b/src/dependencies.pyindex e4ebdf6e5..a4cba9097 100644--- a/src/dependencies.py+++ b/src/dependencies.py@@ -253,7 +253,10 @@ async def get_current_user(# IBM AMS cookie auth takes priority when enabledif IBM_AUTH_ENABLED:logger.debug("[IBM Auth] IBM auth mode enabled, getting current user")- return await _get_ibm_user(request, required=True)+ user = await _get_ibm_user(request, required=True)+ if user and user.user_id and user.user_id not in session_manager.users:+ session_manager.users[user.user_id] = user+ return userif is_no_auth_mode():user = AnonymousUser()@@ -294,7 +297,10 @@ async def get_optional_user(# IBM AMS cookie auth takes priority when enabledif IBM_AUTH_ENABLED:logger.debug("[IBM Auth] IBM auth mode enabled, getting optional user")- return await _get_ibm_user(request, required=False)+ user = await _get_ibm_user(request, required=False)+ if user and user.user_id and user.user_id not in session_manager.users:+ session_manager.users[user.user_id] = user+ return userif is_no_auth_mode():user = AnonymousUser()diff --git a/src/main.py b/src/main.pyindex 1b89a38a9..eba62dc7f 100644--- a/src/main.py+++ b/src/main.py@@ -20,6 +20,7 @@ from utils.encryption import enforce_startup_prerequisitesfrom utils.telemetry import TelemetryClient, Category, MessageIdfrom fastapi import FastAPI, Requestfrom fastapi.responses import JSONResponse+from prometheus_fastapi_instrumentator import Instrumentator# API endpoints@@ -214,12 +215,16 @@ async def _ensure_opensearch_index():# The service can still function, document operations might fail later-async def init_index(opensearch_client=None):+async def init_index(opensearch_client=None, admin_username: str = None):"""Initialize OpenSearch index and security roles"""os_client = opensearch_client or clients.opensearchtry:await wait_for_opensearch(opensearch_client)+ # Initialize OpenSearch security configuration (roles and mapping)+ from utils.opensearch_utils import setup_opensearch_security+ await setup_opensearch_security(os_client, admin_username=admin_username)+# Get the configured embedding model from user configurationconfig = get_openrag_config()embedding_model = config.knowledge.embedding_model@@ -350,13 +355,6 @@ async def init_index(opensearch_client=None):) from eraise e--async def init_index_when_ready(opensearch_client=None):- """Wait for the OpenSearch service to be ready and then initialize the OpenSearch index."""- await wait_for_opensearch(opensearch_client)- await init_index(opensearch_client)--def generate_jwt_keys():"""Generate RSA keys for JWT signing if they don't exist"""from config.paths import get_keys_path@@ -1088,7 +1086,7 @@ async def opensearch_health_ready(request):from config.settings import IBM_AUTH_ENABLED, OPENSEARCH_URLif IBM_AUTH_ENABLED:- logger.debug("[IBM Auth] IBM auth mode enabled, health check per-request")+ logger.debug("[OpenSearch Security] OpenSearch auth mode enabled, health check per-request")# In IBM auth mode we cannot rely on the global OpenSearch client# (auth is established per-request), so perform a lightweight,# unauthenticated connectivity check against the OpenSearch endpoint.@@ -1098,17 +1096,17 @@ async def opensearch_health_ready(request):async with httpx.AsyncClient(timeout=timeout) as client:resp = await client.get(f"{opensearch_url}/")if resp.status_code < 500:- logger.debug("[IBM Auth] OpenSearch health check successful")+ logger.debug("[OpenSearch Security] OpenSearch health check successful")return JSONResponse({"status": "ready","dependencies": {"opensearch": "up"},- "note": "IBM auth mode - connectivity verified via unauthenticated probe",+ "note": "OpenSearch auth mode - connectivity verified via unauthenticated probe",},status_code=200,)else:- logger.debug("[IBM Auth] OpenSearch health check failed")+ logger.debug("[OpenSearch Security] OpenSearch health check failed")return JSONResponse({"status": "not_ready",@@ -1118,7 +1116,7 @@ async def opensearch_health_ready(request):status_code=503,)except Exception as e:- logger.error("[IBM Auth] OpenSearch health check failed", error=str(e))+ logger.error("[OpenSearch Security] OpenSearch health check failed", error=str(e))return JSONResponse({"status": "not_ready",@@ -1135,7 +1133,7 @@ async def opensearch_health_ready(request):status_code=200,)except Exception as e:- logger.error("[IBM Auth] OpenSearch health check failed", error=str(e))+ logger.error("[OpenSearch Security] OpenSearch health check failed", error=str(e))return JSONResponse({"status": "not_ready",@@ -1267,6 +1265,17 @@ async def startup_tasks(services):# Index will be created after onboarding when we know the embedding modelawait wait_for_opensearch()+ # Setup OpenSearch security (roles and mappings) after connection is established+ try:+ from utils.opensearch_utils import setup_opensearch_security+ await setup_opensearch_security(clients.opensearch)+ logger.info("OpenSearch security configuration completed successfully")+ except Exception as e:+ logger.warning(+ "Failed to setup OpenSearch security configuration - continuing anyway",+ error=str(e)+ )+if DISABLE_INGEST_WITH_LANGFLOW:await _ensure_opensearch_index()@@ -1500,6 +1509,11 @@ async def create_app():app = FastAPI(title="OpenRAG API", version=OPENRAG_VERSION, debug=True)app.state.services = services # Store services for cleanupapp.state.background_tasks = set()++ try:+ Instrumentator().instrument(app).expose(app)+ except Exception as e:+ logger.error(f"Failed to instrument app with Prometheus: {str(e)}")# Register route handlers — auth and service injection done via FastAPI Depends() in each handler@@ -2054,15 +2068,25 @@ async def create_app():await TelemetryClient.send_event(Category.APPLICATION_SHUTDOWN, MessageId.ORB_APP_SHUTDOWN)+ logger.info("Application shutdown initiated")++ # Gracefully shutdown OpenSearch connection first+ try:+ from utils.opensearch_utils import graceful_opensearch_shutdown+ await graceful_opensearch_shutdown(clients.opensearch)+ except Exception as e:+ logger.error("Error during graceful OpenSearch shutdown", error=str(e))+await cleanup_subscriptions_proper(services)# Cleanup task service (cancels background tasks and process pool)await services["task_service"].shutdown()- # Cleanup async clients+ # Cleanup async clients (this will also close OpenSearch client if not already closed)await clients.cleanup()# Cleanup telemetry clientfrom utils.telemetry.client import cleanup_telemetry_clientawait cleanup_telemetry_client()+ logger.info("Application shutdown completed")return appdiff --git a/src/session_manager.py b/src/session_manager.pyindex 8185c2146..6832b1f60 100644--- a/src/session_manager.py+++ b/src/session_manager.py@@ -212,6 +212,8 @@ class SessionManager:# Check for token from environment variable firsttoken = os.getenv("OPENSEARCH_JWT_TOKEN")+ if token and (token.startswith("Bearer ") or token.startswith("Basic ")):+ return tokenif not token:token = jwt.encode(token_payload, self.private_key, algorithm=self.algorithm)return f"Bearer {token}"diff --git a/src/utils/opensearch_utils.py b/src/utils/opensearch_utils.pyindex c404b1ca4..57bd0a010 100644--- a/src/utils/opensearch_utils.py+++ b/src/utils/opensearch_utils.py@@ -1,5 +1,7 @@import asyncio+import osimport random+import yamlfrom opensearchpy import AsyncOpenSearchfrom utils.logging_config import get_logger@@ -121,3 +123,265 @@ async def wait_for_opensearch(message: str = "Failed to verify whether OpenSearch is ready."logger.error(message)raise OpenSearchNotReadyError(message)+++async def graceful_opensearch_shutdown(opensearch_client: AsyncOpenSearch) -> None:+ """Gracefully shutdown OpenSearch client connection.++ This ensures that all pending operations are completed and connections+ are properly closed before the application exits.++ Args:+ opensearch_client: The OpenSearch client to shutdown.+ """+ if opensearch_client is None:+ logger.debug("OpenSearch client is None, skipping graceful shutdown")+ return++ try:+ logger.info("Initiating graceful OpenSearch shutdown...")++ # Flush any pending operations by checking cluster health one last time+ try:+ await asyncio.wait_for(+ opensearch_client.cluster.health(),+ timeout=10.0+ )+ logger.debug("Final cluster health check completed")+ except asyncio.TimeoutError:+ logger.warning("Timeout during final cluster health check")+ except Exception as e:+ logger.warning("Error during final cluster health check", error=str(e))++ # Close the client connection+ await opensearch_client.close()+ logger.info("OpenSearch client connection closed gracefully")++ except Exception as e:+ logger.error("Error during graceful OpenSearch shutdown", error=str(e))+++async def setup_opensearch_security(+ opensearch_client: AsyncOpenSearch,+ admin_username: str | None = None,+) -> None:+ """Setup OpenSearch roles and roles mapping.++ The setup involves:+ 1. GET /_plugins/_security/api/rolesmapping (check existing)+ 2. GET /_cluster/health+ 3. PUT /_plugins/_security/api/roles/openrag_user_role (create role)+ 4. PUT /_plugins/_security/api/rolesmapping/openrag_user_role (create mapping)+ 5. PUT /_plugins/_security/api/rolesmapping/all_access (merge admin mapping)+ 6. Verify with final GETs.++ Args:+ opensearch_client: Authenticated OpenSearch client.+ admin_username: OpenSearch username of the onboarding user (IBM mode).+ When provided, this user is pinned into the all_access role mapping's+ ``users`` list so they retain admin access after ``backend_roles``+ are modified for DLS.++ This should be called during initial setup after OpenSearch is ready.+ """+ from config.settings import IBM_AUTH_ENABLED++ logger.info("Initializing OpenSearch security configuration...", ibm_auth=IBM_AUTH_ENABLED)++ base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))+ if IBM_AUTH_ENABLED:+ security_config_dir = os.path.join(base_dir, "cloud_securityconfig")+ else:+ security_config_dir = os.path.join(base_dir, "securityconfig")++ logger.info("[OpenSearch Security] Using config directory", config_dir=security_config_dir)++ roles_file = os.path.join(security_config_dir, "roles.yml")+ roles_mapping_file = os.path.join(security_config_dir, "roles_mapping.yml")++ logger.info(+ "[OpenSearch Security] Configuration paths",+ base_dir=base_dir,+ security_config_dir=security_config_dir,+ roles_file=roles_file,+ roles_mapping_file=roles_mapping_file,+ )++ try:+ # 1. & 2. Readiness checks+ logger.info("[OpenSearch Security] Performing readiness checks...")++ try:+ rolesmapping_response = await opensearch_client.transport.perform_request("GET", "/_plugins/_security/api/rolesmapping")+ logger.info("[OpenSearch Security] Current rolesmapping retrieved", count=len(rolesmapping_response) if isinstance(rolesmapping_response, dict) else "unknown")+ except Exception as e:+ logger.warning("[OpenSearch Security] Failed to get current rolesmapping", error=str(e))++ cluster_health = await opensearch_client.cluster.health()+ logger.info("[OpenSearch Security] Cluster health check passed", status=cluster_health.get("status"))++ # Load role definitions from YAML+ if not os.path.exists(roles_file):+ logger.error(f"[OpenSearch Security] Roles configuration file not found: {roles_file}")+ raise FileNotFoundError(f"Roles configuration file not found: {roles_file}")++ with open(roles_file, "r") as f:+ roles_config = yaml.safe_load(f)++ logger.info("[OpenSearch Security] Loaded roles configuration", roles=list(roles_config.keys()) if roles_config else [])++ # 3. Create openrag_user_role+ if "openrag_user_role" in roles_config:+ role_body = roles_config["openrag_user_role"]++ logger.info(+ "[OpenSearch Security] Creating 'openrag_user_role' role",+ patterns=role_body['index_permissions'][0]['index_patterns'] if 'index_permissions' in role_body else 'default',+ allowed_actions=role_body['index_permissions'][0].get('allowed_actions', []) if 'index_permissions' in role_body else []+ )++ resp = await opensearch_client.transport.perform_request(+ "PUT",+ "/_plugins/_security/api/roles/openrag_user_role",+ body=role_body,+ headers={"Content-Type": "application/json"}+ )+ logger.info("[OpenSearch Security] Role creation response", response=resp)+ else:+ logger.warning("[OpenSearch Security] 'openrag_user_role' not found in roles.yml")++ # Load roles mapping from YAML+ if not os.path.exists(roles_mapping_file):+ logger.error(f"[OpenSearch Security] Roles mapping file not found: {roles_mapping_file}")+ raise FileNotFoundError(f"Roles mapping file not found: {roles_mapping_file}")++ with open(roles_mapping_file, "r") as f:+ mapping_config = yaml.safe_load(f)++ logger.info("[OpenSearch Security] Loaded roles mapping configuration", mappings=list(mapping_config.keys()) if mapping_config else [])++ # 4. Create openrag_user_role mapping+ if "openrag_user_role" in mapping_config:+ mapping_body = mapping_config["openrag_user_role"]+ logger.info(+ "[OpenSearch Security] Creating 'openrag_user_role' mapping",+ backend_roles=mapping_body.get("backend_roles", []),+ users=mapping_body.get("users", [])+ )+ resp = await opensearch_client.transport.perform_request(+ "PUT",+ "/_plugins/_security/api/rolesmapping/openrag_user_role",+ body=mapping_body,+ headers={"Content-Type": "application/json"}+ )+ logger.info("[OpenSearch Security] Role mapping update response", response=resp)++ # 5. Update all_access mapping — merge with existing to preserve+ # IBM-managed entries, but ensure backend_roles never contains+ # "all_access" (which would give IBM API key users the super-admin+ # role and bypass DLS).+ if "all_access" in mapping_config:+ all_access_body = mapping_config["all_access"]++ if "backend_roles" not in all_access_body:+ all_access_body["backend_roles"] = ["admin"]+ if "description" not in all_access_body:+ all_access_body["description"] = "Maps admin to all_access"++ # Always fetch existing mapping first so we never lose previous admins+ # in multi-tenant deployments where each tenant onboards independently.+ existing_users: list = []+ existing_hosts: list = []+ existing_backend_roles: list = []+ try:+ existing = await opensearch_client.transport.perform_request(+ "GET", "/_plugins/_security/api/rolesmapping/all_access"+ )+ existing_mapping = existing.get("all_access", {})+ existing_users = existing_mapping.get("users", []) or []+ existing_hosts = existing_mapping.get("hosts", []) or []+ existing_backend_roles = existing_mapping.get("backend_roles", []) or []+ except Exception:+ logger.debug("[OpenSearch Security] No existing all_access mapping found, creating fresh")++ # Build merged users: source file + cluster + new admin (bare + ibmlhapikey_ variant).+ # Adding both variants ensures the user can authenticate via JWT *and* via IBM+ # Basic-Auth (ibmlhapikey_<username>), which are treated as separate principals+ # by OpenSearch's security plugin.+ new_admin_users: list = []+ if IBM_AUTH_ENABLED and admin_username:+ new_admin_users = [admin_username, f"ibmlhapikey_{admin_username}"]+ logger.info(+ "[OpenSearch Security] Pinning onboarding user as admin (both variants)",+ users=new_admin_users,+ )++ merged_users = list(set(+ all_access_body.get("users", []) + existing_users + new_admin_users+ ))+ all_access_body["users"] = merged_users+ logger.debug("[OpenSearch Security] Merged all_access users", users=merged_users)++ if existing_hosts:+ merged_hosts = list(set(all_access_body.get("hosts", []) + existing_hosts))+ all_access_body["hosts"] = merged_hosts+ logger.debug(+ "[OpenSearch Security] Preserved existing all_access hosts",+ hosts=merged_hosts,+ )++ if existing_backend_roles:+ safe_existing_backend_roles = [+ r for r in existing_backend_roles if r != "all_access"+ ]+ merged_backend_roles = list(+ set(all_access_body.get("backend_roles", []) + safe_existing_backend_roles)+ )+ all_access_body["backend_roles"] = merged_backend_roles+ logger.debug(+ "[OpenSearch Security] Preserved existing all_access backend_roles",+ backend_roles=merged_backend_roles,+ )++ if "all_access" in all_access_body.get("backend_roles", []):+ all_access_body["backend_roles"] = [+ r for r in all_access_body["backend_roles"] if r != "all_access"+ ]+ logger.info(+ "[OpenSearch Security] Removed 'all_access' from all_access backend_roles to preserve DLS",+ final_backend_roles=all_access_body["backend_roles"],+ )++ logger.info("[OpenSearch Security] Updating 'all_access' mapping...", body=all_access_body)+ resp = await opensearch_client.transport.perform_request(+ "PUT",+ "/_plugins/_security/api/rolesmapping/all_access",+ body=all_access_body,+ headers={"Content-Type": "application/json"}+ )+ logger.info("[OpenSearch Security] All access mapping update response", response=resp)++ # 6. Final verification+ logger.info("[OpenSearch Security] Verifying security configuration...")+ role_verify = await opensearch_client.transport.perform_request("GET", "/_plugins/_security/api/roles/openrag_user_role")+ logger.info("[OpenSearch Security] Role verification", role=role_verify)++ mapping_verify = await opensearch_client.transport.perform_request("GET", "/_plugins/_security/api/rolesmapping/openrag_user_role")+ logger.info("[OpenSearch Security] Role mapping verification", mapping=mapping_verify)++ logger.info("Successfully completed OpenSearch security configuration.")++ except Exception as e:+ # Check for authentication errors or if the security plugin is missing+ error_str = str(e).lower()+ if any(code in error_str for code in ["401", "403", "404", "security_exception", "not_found"]):+ logger.warning(+ "Skipping OpenSearch security configuration: "+ "The cluster may not have the security plugin enabled or "+ "the provided credentials do not have administrative permissions."+ )+ return++ logger.error("Failed to setup OpenSearch security configuration", error=str(e))+ # Re-raise for non-auth/non-security errors to ensure visibility+ raisediff --git a/tests/unit/test_opensearch_security_setup.py b/tests/unit/test_opensearch_security_setup.pynew file mode 100644index 000000000..fa3ea60ee--- /dev/null+++ b/tests/unit/test_opensearch_security_setup.py@@ -0,0 +1,69 @@+import pytest+from unittest.mock import AsyncMock, patch, MagicMock++from utils.opensearch_utils import setup_opensearch_security++async def test_setup_opensearch_security_success():+ """Test successful security setup with all expected calls."""+ mock_client = MagicMock()+ mock_client.transport.perform_request = AsyncMock(return_value={"status": "OK", "message": "Success"})+ mock_client.cluster.health = AsyncMock(return_value={"status": "green"})++ # Sample configurations+ roles_data = {+ "openrag_user_role": {+ "cluster_permissions": ["read"],+ "index_permissions": [{"index_patterns": ["*"], "allowed_actions": ["crud"]}]+ }+ }+ mapping_data = {+ "openrag_user_role": {"backend_roles": ["openrag_user"]},+ "all_access": {"users": ["admin"]}+ }++ # Mock file existence and content+ with patch("os.path.exists", return_value=True), \+ patch("builtins.open", MagicMock()), \+ patch("yaml.safe_load") as mock_yaml, \+ patch("utils.opensearch_utils.get_index_name", return_value="dynamic_index"):++ mock_yaml.side_effect = [roles_data, mapping_data]++ await setup_opensearch_security(mock_client)++ # Verify calls+ assert mock_client.transport.perform_request.call_count == 6+ mock_client.cluster.health.assert_called_once()++ # Check the role creation body for dynamic patterns+ role_put_call = mock_client.transport.perform_request.call_args_list[1]+ role_body = role_put_call[1]['body']+ patterns = role_body['index_permissions'][0]['index_patterns']+ assert "dynamic_index" in patterns+ assert "dynamic_index*" in patterns+ assert "knowledge_filters" in patterns+ assert "knowledge_filters*" in patterns++async def test_setup_opensearch_security_graceful_auth_error():+ """Test that auth/security errors are handled gracefully without raising."""+ mock_client = MagicMock()+ # Mock a 401 Unauthorized error+ mock_client.transport.perform_request = AsyncMock(side_effect=Exception("401 Unauthorized"))++ # This should NOT raise an exception+ await setup_opensearch_security(mock_client)+ assert mock_client.transport.perform_request.call_count == 1++async def test_setup_opensearch_security_missing_files():+ """Test that missing configuration files raise FileNotFoundError."""+ mock_client = MagicMock()+ mock_client.transport.perform_request = AsyncMock()+ mock_client.cluster.health = AsyncMock()++ with patch("os.path.exists", return_value=False), \+ patch("utils.opensearch_utils.get_index_name", return_value="docs"):+ with pytest.raises(FileNotFoundError):+ await setup_opensearch_security(mock_client)diff --git a/uv.lock b/uv.lockindex 469a857be..53dc1ffba 100644--- a/uv.lock+++ b/uv.lock@@ -2,11 +2,17 @@ version = 1revision = 3requires-python = ">=3.13"resolution-markers = [- "platform_machine == 'x86_64' and sys_platform == 'linux'",- "platform_machine == 'aarch64' and sys_platform == 'linux'",+ "python_full_version >= '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux'",+ "python_full_version < '3.14' and platform_machine == 'x86_64' and sys_platform == 'linux'",+ "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'",+ "python_full_version < '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'","python_version < '0'",- "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')",- "sys_platform == 'darwin'",+ "python_full_version >= '3.14' and sys_platform == 'win32'",+ "python_full_version < '3.14' and sys_platform == 'win32'",+ "(python_full_version >= '3.14' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')",+ "(python_full_version < '3.14' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')",+ "python_full_version >= '3.14' and sys_platform == 'darwin'",+ "python_full_version < '3.14' and sys_platform == 'darwin'",][[package]]@@ -982,6 +988,7 @@ dependencies = [{ name = "msal" },{ name = "openai" },{ name = "opensearch-py", extra = ["async"] },+ { name = "prometheus-fastapi-instrumentator" },{ name = "psutil" },{ name = "pyjwt" },{ name = "python-dotenv" },@@ -1018,6 +1025,7 @@ requires-dist = [{ name = "msal", specifier = ">=1.29.0" },{ name = "openai", specifier = ">=1.0.0" },{ name = "opensearch-py", extras = ["async"], specifier = ">=3.0.0" },+ { name = "prometheus-fastapi-instrumentator", specifier = ">=7.1.0" },{ name = "psutil", specifier = ">=7.0.0" },{ name = "pyjwt", specifier = ">=2.8.0" },{ name = "python-dotenv", specifier = ">=1.0.0" },@@ -1101,6 +1109,28 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },]+[[package]]+name = "prometheus-client"+version = "0.24.1"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/f0/58/a794d23feb6b00fc0c72787d7e87d872a6730dd9ed7c7b3e954637d8f280/prometheus_client-0.24.1.tar.gz", hash = "sha256:7e0ced7fbbd40f7b84962d5d2ab6f17ef88a72504dcf7c0b40737b43b2a461f9", size = 85616, upload-time = "2026-01-14T15:26:26.965Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/74/c3/24a2f845e3917201628ecaba4f18bab4d18a337834c1df2a159ee9d22a42/prometheus_client-0.24.1-py3-none-any.whl", hash = "sha256:150db128af71a5c2482b36e588fc8a6b95e498750da4b17065947c16070f4055", size = 64057, upload-time = "2026-01-14T15:26:24.42Z" },+]++[[package]]+name = "prometheus-fastapi-instrumentator"+version = "7.1.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "prometheus-client" },+ { name = "starlette" },+]+sdist = { url = "https://files.pythonhosted.org/packages/69/6d/24d53033cf93826aa7857699a4450c1c67e5b9c710e925b1ed2b320c04df/prometheus_fastapi_instrumentator-7.1.0.tar.gz", hash = "sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e", size = 20220, upload-time = "2025-03-19T19:35:05.351Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/27/72/0824c18f3bc75810f55dacc2dd933f6ec829771180245ae3cc976195dec0/prometheus_fastapi_instrumentator-7.1.0-py3-none-any.whl", hash = "sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9", size = 19296, upload-time = "2025-03-19T19:35:04.323Z" },+]+[[package]]name = "propcache"version = "0.4.1"