OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: shared documents ingestion mode for IBM COS connectors (#1954)
Rico Furtado74c5c117143b
diff --git a/alembic/versions/0007_add_knowledge_delete_anonymous.py b/alembic/versions/0007_add_knowledge_delete_anonymous.pynew file mode 100644index 000000000..c112e73d5--- /dev/null+++ b/alembic/versions/0007_add_knowledge_delete_anonymous.py@@ -0,0 +1,92 @@+"""add knowledge:delete:anonymous permission++Revision ID: 0007_add_knowledge_delete_anonymous+Revises: 0006_revoke_provider_override_nonadmin+Create Date: 2026-06-11 00:00:00.000000++Adds a dedicated permission for deleting ownerless shared documents and grants+it to the built-in admin role. The existing knowledge:delete:any permission is+left unchanged.+"""++import uuid+from collections.abc import Sequence++import sqlalchemy as sa++from alembic import op++revision: str = "0007_add_knowledge_delete_anonymous"+down_revision: str | Sequence[str] | None = "0006_revoke_provider_override_nonadmin"+branch_labels: str | Sequence[str] | None = None+depends_on: str | Sequence[str] | None = None++_PERMISSION_NAME = "knowledge:delete:anonymous"+_ADMIN_ROLE_NAME = "admin"+++def upgrade() -> None:+ bind = op.get_bind()+ permission_id = bind.execute(+ sa.text("SELECT id FROM permissions WHERE name = :name"),+ {"name": _PERMISSION_NAME},+ ).scalar()++ if permission_id is None:+ permission_id = str(uuid.uuid4())+ bind.execute(+ sa.text(+ "INSERT INTO permissions "+ "(id, name, resource, action, description) "+ "VALUES (:id, :name, :resource, :action, :description)"+ ),+ {+ "id": permission_id,+ "name": _PERMISSION_NAME,+ "resource": "knowledge",+ "action": "delete:anonymous",+ "description": "Delete anonymous shared documents",+ },+ )++ admin_role_id = bind.execute(+ sa.text("SELECT id FROM roles WHERE name = :name"),+ {"name": _ADMIN_ROLE_NAME},+ ).scalar()+ if admin_role_id is None:+ return++ existing = bind.execute(+ sa.text(+ "SELECT 1 FROM role_permissions "+ "WHERE role_id = :role_id AND permission_id = :permission_id"+ ),+ {"role_id": admin_role_id, "permission_id": permission_id},+ ).first()+ if existing is None:+ bind.execute(+ sa.text(+ "INSERT INTO role_permissions (role_id, permission_id) "+ "VALUES (:role_id, :permission_id)"+ ),+ {"role_id": admin_role_id, "permission_id": permission_id},+ )+++def downgrade() -> None:+ bind = op.get_bind()+ permission_id = bind.execute(+ sa.text("SELECT id FROM permissions WHERE name = :name"),+ {"name": _PERMISSION_NAME},+ ).scalar()+ if permission_id is None:+ return++ bind.execute(+ sa.text("DELETE FROM role_permissions WHERE permission_id = :permission_id"),+ {"permission_id": permission_id},+ )+ bind.execute(+ sa.text("DELETE FROM permissions WHERE id = :permission_id"),+ {"permission_id": permission_id},+ )diff --git a/frontend/app/api/mutations/useSyncConnector.ts b/frontend/app/api/mutations/useSyncConnector.tsindex 287aa3e5d..3017d692c 100644--- a/frontend/app/api/mutations/useSyncConnector.ts+++ b/frontend/app/api/mutations/useSyncConnector.ts@@ -77,6 +77,8 @@ const syncConnector = async ({bucket_filter?: string[];/** When true, replace any indexed document with the same filename. */replace_duplicates?: boolean;+ /** When true (COS only), index without an owner so all users in the instance can retrieve the document. */+ shared?: boolean;};}): Promise<SyncResponse> => {const response = await fetch(`/api/connectors/${connectorType}/sync`, {diff --git a/frontend/components/cloud-picker/ingest-settings.tsx b/frontend/components/cloud-picker/ingest-settings.tsxindex ae56d9da8..ceb007bbd 100644--- a/frontend/components/cloud-picker/ingest-settings.tsx+++ b/frontend/components/cloud-picker/ingest-settings.tsx@@ -42,6 +42,8 @@ interface IngestSettingsProps {onOpenChange: (open: boolean) => void;settings?: IngestSettingsType;onSettingsChange?: (settings: IngestSettingsType) => void;+ /** When true, show the "Make documents available to all users" toggle. COS ingestion only. */+ showShared?: boolean;}export const IngestSettings = ({@@ -49,6 +51,7 @@ export const IngestSettings = ({onOpenChange,settings,onSettingsChange,+ showShared = false,}: IngestSettingsProps) => {const { isAuthenticated, isNoAuthMode } = useAuth();@@ -244,7 +247,13 @@ export const IngestSettings = ({/></div>- <div className="flex items-center justify-between">+ <div+ className={+ showShared+ ? "flex items-center justify-between border-b pb-3 mb-3"+ : "flex items-center justify-between"+ }+ ><div><div className="text-sm pb-2 font-semibold">Picture descriptions@@ -260,6 +269,26 @@ export const IngestSettings = ({}/></div>++ {showShared && (+ <div className="flex items-center justify-between">+ <div>+ <div className="text-sm pb-2 font-semibold">+ Make documents available to all users+ </div>+ <div className="text-sm text-muted-foreground">+ Shared documents are visible to all users in this OpenRAG+ instance.+ </div>+ </div>+ <Switch+ checked={currentSettings.shared ?? false}+ onCheckedChange={(checked) =>+ handleSettingsChange({ shared: checked })+ }+ />+ </div>+ )}</div></CollapsibleContent></Collapsible>diff --git a/frontend/components/cloud-picker/types.ts b/frontend/components/cloud-picker/types.tsindex f3eea21f5..fc125cbef 100644--- a/frontend/components/cloud-picker/types.ts+++ b/frontend/components/cloud-picker/types.ts@@ -113,6 +113,8 @@ export interface IngestSettings {ocr: boolean;pictureDescriptions: boolean;embeddingModel: string;+ /** When true, index without an owner so all users in the instance can retrieve the document. COS only. */+ shared?: boolean;}/** Inline error message if chunk settings are invalid; otherwise null. */diff --git a/frontend/components/connectors/shared-bucket-view.tsx b/frontend/components/connectors/shared-bucket-view.tsxindex e6b91b94e..ab67d7f26 100644--- a/frontend/components/connectors/shared-bucket-view.tsx+++ b/frontend/components/connectors/shared-bucket-view.tsx@@ -28,6 +28,8 @@ export interface SharedBucketViewProps {) => void;onBack: () => void;onDone: () => void;+ /** When true, show the "Make documents available to all users" toggle. COS ingestion only. */+ showShared?: boolean;}export function SharedBucketView({@@ -41,6 +43,7 @@ export function SharedBucketView({addTask,onBack,onDone,+ showShared = false,}: SharedBucketViewProps) {const queryClient = useQueryClient();const { isAuthenticated, isNoAuthMode } = useAuth();@@ -97,6 +100,7 @@ export function SharedBucketView({selected_files: [],bucket_filter: Array.from(selectedBuckets),settings: ingestSettings,+ shared: showShared ? (ingestSettings.shared ?? false) : undefined,},},{@@ -272,6 +276,7 @@ export function SharedBucketView({onOpenChange={setIsSettingsOpen}settings={ingestSettings}onSettingsChange={setIngestSettings}+ showShared={showShared}/>)}</div>diff --git a/frontend/components/knowledge-actions-dropdown.tsx b/frontend/components/knowledge-actions-dropdown.tsxindex 29330a72a..f92e30569 100644--- a/frontend/components/knowledge-actions-dropdown.tsx+++ b/frontend/components/knowledge-actions-dropdown.tsx@@ -208,7 +208,7 @@ export const KnowledgeActionsDropdown = ({</TooltipProvider>)}<RequirePermission- anyOf={["knowledge:delete:own", "knowledge:delete:any"]}+ anyOf={["knowledge:delete:own", "knowledge:delete:anonymous"]}><DropdownMenuItemclassName="text-destructive focus:text-destructive cursor-pointer"diff --git a/frontend/components/knowledge-batch-actions-bar.tsx b/frontend/components/knowledge-batch-actions-bar.tsxindex b01b1e407..923c9b094 100644--- a/frontend/components/knowledge-batch-actions-bar.tsx+++ b/frontend/components/knowledge-batch-actions-bar.tsx@@ -15,7 +15,10 @@ export const KnowledgeBatchActionsBar = ({onCancel,}: KnowledgeBatchActionsBarProps) => {const { canAny } = usePermissions();- const canDelete = canAny(["knowledge:delete:own", "knowledge:delete:any"]);+ const canDelete = canAny([+ "knowledge:delete:own",+ "knowledge:delete:anonymous",+ ]);return (<div className="flex h-12 w-full items-stretch bg-primary text-primary-foreground"><buttondiff --git a/frontend/enhancements/connectors/ibm-cos/components/bucket-view.tsx b/frontend/enhancements/connectors/ibm-cos/components/bucket-view.tsxindex 326c38b3e..870ba651a 100644--- a/frontend/enhancements/connectors/ibm-cos/components/bucket-view.tsx+++ b/frontend/enhancements/connectors/ibm-cos/components/bucket-view.tsx@@ -37,6 +37,7 @@ export function IBMCOSBucketView({addTask={addTask}onBack={onBack}onDone={onDone}+ showShared/>);}diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 9ca877b82..901148957 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -11,7 +11,9 @@ from dependencies import (get_connector_service,get_current_user,get_db_session,+ get_rbac_service,get_session_manager,+ has_effective_permission,require_permission,)from services.connector_access_service import (@@ -386,6 +388,10 @@ class ConnectorSyncBody(BaseModel):# rather than failing. Set by the provider upload UI after the user confirms# overwrite in the duplicate dialog.replace_duplicates: bool = False+ # When True (COS only), index chunks without an owner field so OpenSearch DLS+ # makes them visible to all users in the instance. Temporary CIO mechanism;+ # not a full ACL feature. Defaults to False (private).+ shared: bool = Falseclass ConnectorCheckDuplicatesBody(BaseModel):@@ -643,6 +649,7 @@ async def connector_sync(session_manager=Depends(get_session_manager),user: User = Depends(require_permission("connectors:use")),session: AsyncSession = Depends(get_db_session),+ rbac=Depends(get_rbac_service),):"""Sync files from all active connections of a connector type"""if denied := await _connector_access_denied(request, session, connector_type):@@ -672,6 +679,12 @@ async def connector_sync()jwt_token = user.jwt_token+ if body.shared and connector_type != "ibm_cos":+ return JSONResponse(+ {"error": "shared flag is only supported for the ibm_cos connector"},+ status_code=400,+ )+# Get all active connections for this connector type and userconnections = await connector_service.connection_manager.list_connections(user_id=user.user_id, connector_type=connector_type@@ -730,6 +743,17 @@ async def connector_sync(# Explicit files selected (e.g., from file picker) - sync those specific filesfrom .documents import _ensure_index_exists+ if body.shared and body.replace_duplicates:+ if not await has_effective_permission(+ request, user, rbac, "knowledge:delete:anonymous"+ ):+ return JSONResponse(+ {+ "error": "Replacing shared documents requires the knowledge:delete:anonymous permission"+ },+ status_code=403,+ )+await _ensure_index_exists(jwt_token)task_id = await connector_service.sync_specific_files(working_connection.connection_id,@@ -739,6 +763,7 @@ async def connector_sync(file_infos=file_infos,ingest_settings=body.settings,replace_duplicates=body.replace_duplicates,+ shared=body.shared,)elif body.sync_all or body.bucket_filter:# Full ingest: discover and ingest all files (or files from specific buckets).@@ -780,6 +805,7 @@ async def connector_sync(all_file_ids,jwt_token=jwt_token,ingest_settings=body.settings,+ shared=body.shared,)else:# sync_all: ingest everything the connector can see@@ -789,6 +815,7 @@ async def connector_sync(max_files=max_files,jwt_token=jwt_token,ingest_settings=body.settings,+ shared=body.shared,)else:# No files specified - sync only files already in OpenSearch for this connector@@ -854,6 +881,7 @@ async def connector_sync(jwt_token=jwt_token,ingest_settings=body.settings,replace_duplicates=_connector_sync_should_replace(connector_type),+ shared=body.shared,)else:# Fallback: use filename filtering (for Langflow-ingested files without document_id)@@ -870,6 +898,7 @@ async def connector_sync(filename_filter=set(existing_filenames),ingest_settings=body.settings,replace_duplicates=_connector_sync_should_replace(connector_type),+ shared=body.shared,)task_ids = [task_id]await TelemetryClient.send_event(diff --git a/src/api/documents.py b/src/api/documents.pyindex 227b1edc3..c614dd7c1 100644--- a/src/api/documents.py+++ b/src/api/documents.py@@ -1,13 +1,23 @@-from fastapi import Depends+from fastapi import Depends, Requestfrom fastapi.responses import JSONResponsefrom pydantic import BaseModel-from dependencies import get_current_user, get_session_manager, require_permission+from dependencies import (+ get_current_user,+ get_rbac_service,+ get_session_manager,+ has_effective_permission,+ require_any_permission,+)from session_manager import Userfrom utils.logging_config import get_loggerlogger = get_logger(__name__)+require_document_delete_permission = require_any_permission(+ ("knowledge:delete:own", "knowledge:delete:anonymous")+)+class DeleteDocumentBody(BaseModel):filename: str@@ -18,11 +28,18 @@ async def delete_documents_by_filename_core(session_manager,user_id: str,jwt_token: str | None,+ can_delete_own: bool = True,+ can_delete_anonymous: bool = False,):"""Shared delete-by-filename logic for v1 and non-v1 endpoints."""from config.settings import get_index_namefrom utils.opensearch_delete import collect_visible_document_ids, delete_document_ids- from utils.opensearch_queries import build_filename_query, build_owned_filename_query+ from utils.opensearch_queries import (+ build_anonymous_filename_query,+ build_filename_query,+ build_owned_filename_query,+ build_replace_filename_query,+ )normalized_filename = (filename or "").strip()if not normalized_filename:@@ -38,16 +55,58 @@ async def delete_documents_by_filename_core()try:- opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)index_name = get_index_name()+ from config.settings import clients- owned_document_ids = await collect_visible_document_ids(- opensearch_client,- index=index_name,- query=build_owned_filename_query(normalized_filename, user_id),- )+ write_client = clients.opensearch- if not owned_document_ids:+ if can_delete_own and can_delete_anonymous:+ if write_client is None:+ raise RuntimeError("Backend OpenSearch client is unavailable")+ document_ids = await collect_visible_document_ids(+ write_client,+ index=index_name,+ query=build_replace_filename_query(normalized_filename, user_id),+ )+ elif can_delete_anonymous:+ if write_client is None:+ raise RuntimeError("Backend OpenSearch client is unavailable")+ document_ids = await collect_visible_document_ids(+ write_client,+ index=index_name,+ query=build_anonymous_filename_query(normalized_filename),+ )+ elif can_delete_own:+ opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)+ document_ids = await collect_visible_document_ids(+ opensearch_client,+ index=index_name,+ query=build_owned_filename_query(normalized_filename, user_id),+ )+ else:+ return (+ {+ "success": False,+ "deleted_chunks": 0,+ "filename": normalized_filename,+ "message": None,+ "error": "Access denied: insufficient permissions",+ },+ 403,+ )++ if not document_ids:+ if can_delete_anonymous:+ return (+ {+ "success": False,+ "deleted_chunks": 0,+ "filename": normalized_filename,+ "message": None,+ "error": "No matching document chunks were deleted. The file may be missing or not deletable in the current user context.",+ },+ 404,+ )visible_check = await opensearch_client.search(index=index_name,body={@@ -79,12 +138,12 @@ async def delete_documents_by_filename_core(404,)- from config.settings import clients-+ if write_client is None:+ raise RuntimeError("Backend OpenSearch client is unavailable")deleted_count = await delete_document_ids(- clients.opensearch,+ write_client,index=index_name,- document_ids=owned_document_ids,+ document_ids=document_ids,)logger.info(f"Deleted {deleted_count} chunks for filename {normalized_filename}",@@ -243,14 +302,30 @@ async def check_filename_exists(async def delete_documents_by_filename(body: DeleteDocumentBody,+ request: Request,session_manager=Depends(get_session_manager),- user: User = Depends(require_permission("knowledge:delete:own")),+ rbac=Depends(get_rbac_service),+ user: User = Depends(require_document_delete_permission),):"""Delete all documents with a specific filename"""+ can_delete_own = await has_effective_permission(+ request,+ user,+ rbac,+ "knowledge:delete:own",+ )+ can_delete_anonymous = await has_effective_permission(+ request,+ user,+ rbac,+ "knowledge:delete:anonymous",+ )payload, status_code = await delete_documents_by_filename_core(filename=body.filename,session_manager=session_manager,user_id=user.user_id,jwt_token=user.jwt_token,+ can_delete_own=can_delete_own,+ can_delete_anonymous=can_delete_anonymous,)return JSONResponse(payload, status_code=status_code)diff --git a/src/api/v1/documents.py b/src/api/v1/documents.pyindex dd108b259..07f1f5203 100644--- a/src/api/v1/documents.py+++ b/src/api/v1/documents.py@@ -5,7 +5,7 @@ Provides document ingestion and management.Uses API key authentication."""-from fastapi import Depends, File, Form, UploadFile+from fastapi import Depends, File, Form, Request, UploadFilefrom fastapi.responses import JSONResponsefrom pydantic import BaseModel@@ -16,8 +16,11 @@ from dependencies import (get_document_service,get_knowledge_filter_service,get_langflow_file_service,+ get_rbac_service,get_session_manager,get_task_service,+ has_effective_permission,+ require_api_key_any_permission,require_api_key_permission,)from session_manager import User@@ -25,6 +28,10 @@ from utils.logging_config import get_loggerlogger = get_logger(__name__)+require_document_delete_permission = require_api_key_any_permission(+ ("knowledge:delete:own", "knowledge:delete:anonymous")+)+class DeleteDocV1Body(BaseModel):filename: str | None = None@@ -128,8 +135,10 @@ async def task_status_enhanced_endpoint(async def delete_document_endpoint(body: DeleteDocV1Body,+ request: Request,session_manager=Depends(get_session_manager),- user: User = Depends(require_api_key_permission("knowledge:delete:own")),+ rbac=Depends(get_rbac_service),+ user: User = Depends(require_document_delete_permission),knowledge_filter_service=Depends(get_knowledge_filter_service),):"""Delete document(s) from the knowledge base. DELETE /v1/documents@@ -146,6 +155,19 @@ async def delete_document_endpoint(status_code=400,)+ can_delete_own = await has_effective_permission(+ request,+ user,+ rbac,+ "knowledge:delete:own",+ )+ can_delete_anonymous = await has_effective_permission(+ request,+ user,+ rbac,+ "knowledge:delete:anonymous",+ )+if body.filter_id:resolved = await resolve_filter_id(body.filter_id,@@ -169,19 +191,24 @@ async def delete_document_endpoint(session_manager=session_manager,user_id=user.user_id,jwt_token=user.jwt_token,+ can_delete_own=can_delete_own,+ can_delete_anonymous=can_delete_anonymous,)results.append(payload)statuses.append(_status)total_deleted += payload.get("deleted_chunks", 0) or 0+ all_ok = all(200 <= status < 300 for status in statuses)+ response_status = 200 if all_ok else max(statuses)return JSONResponse({- "success": all(200 <= status < 300 for status in statuses),+ "success": all_ok,"deleted_chunks": total_deleted,"filenames": filenames,"filter_id": body.filter_id,"per_file": results,- }+ },+ status_code=response_status,)payload, status_code = await delete_documents_by_filename_core(@@ -189,5 +216,7 @@ async def delete_document_endpoint(session_manager=session_manager,user_id=user.user_id,jwt_token=user.jwt_token,+ can_delete_own=can_delete_own,+ can_delete_anonymous=can_delete_anonymous,)return JSONResponse(payload, status_code=status_code)diff --git a/src/connectors/service.py b/src/connectors/service.pyindex 3086b893b..dec89e118 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -348,6 +348,7 @@ class ConnectorService:filename_filter: set = None,ingest_settings: dict[str, Any] | None = None,replace_duplicates: bool = False,+ shared: bool = False,) -> str:"""Sync files from a connector connection using existing task tracking system.@@ -385,6 +386,9 @@ class ConnectorService:if not connector.is_authenticated:raise ValueError(f"Connection '{connection_id}' not authenticated")+ if shared and connector.CONNECTOR_TYPE != "ibm_cos":+ raise ValueError("shared flag is only supported for the ibm_cos connector")+# Collect files to process (limited by max_files)files_to_process: list[dict[str, Any]] = []page_token = None@@ -450,6 +454,7 @@ class ConnectorService:ingest_settings=ingest_settings,replace_duplicates=replace_duplicates,connector_type=connector.CONNECTOR_TYPE,+ shared=shared,)# Use file IDs as items (no more fake file paths!)@@ -478,6 +483,7 @@ class ConnectorService:file_infos: list[dict[str, Any]] = None,ingest_settings: dict[str, Any] | None = None,replace_duplicates: bool = False,+ shared: bool = False,) -> str:"""Sync specific files by their IDs (used for webhook-triggered syncs or manual selection).@@ -507,6 +513,9 @@ class ConnectorService:if not connector.is_authenticated:raise ValueError(f"Connection '{connection_id}' not authenticated")+ if shared and connector.CONNECTOR_TYPE != "ibm_cos":+ raise ValueError("shared flag is only supported for the ibm_cos connector")+if not file_ids:raise ValueError("No file IDs provided")@@ -633,6 +642,7 @@ class ConnectorService:ingest_settings=ingest_settings,replace_duplicates=replace_duplicates,connector_type=connector.CONNECTOR_TYPE,+ shared=shared,)# Create custom task using TaskServicediff --git a/src/db/seed.py b/src/db/seed.pyindex 625bc204d..6e727dcd7 100644--- a/src/db/seed.py+++ b/src/db/seed.py@@ -56,6 +56,7 @@ PERMISSIONS: list[tuple[str, str, str]] = [("knowledge", "upload", "Upload documents"),("knowledge", "delete:own", "Delete own documents"),("knowledge", "delete:any", "Delete any document"),+ ("knowledge", "delete:anonymous", "Delete anonymous shared documents"),("knowledge", "read:own", "Read own documents"),("knowledge", "read:all", "Read all documents"),("kf", "create", "Create knowledge filters"),diff --git a/src/dependencies.py b/src/dependencies.pyindex 6f36add51..320aadcaa 100644--- a/src/dependencies.py+++ b/src/dependencies.py@@ -425,6 +425,38 @@ def require_permission(perm: str):return _dep+def require_any_permission(required_perms: Sequence[str]):+ """Require at least one permission for a browser-authenticated request."""+ required = tuple(required_perms)+ if not required:+ raise ValueError("require_any_permission requires at least one permission")++ from services.rbac_service import is_rbac_enforced++ async def _dep(+ request: Request,+ user: User = Depends(get_current_user),+ rbac=Depends(get_rbac_service),+ ) -> User:+ if not is_rbac_enforced():+ return await _attach_db_user_id(request, user)+ role_override = getattr(request.state, "api_key_role_ids", None)+ db_user_id = await _resolve_db_user_id(user)+ user = dataclasses.replace(user, db_user_id=db_user_id)+ request.state.db_user_id = db_user_id+ request.state.user = user+ perms = await rbac.get_user_permissions(db_user_id, role_override=role_override)+ if not any(perm in perms for perm in required):+ await rbac.audit_denied(db_user_id, "|".join(required))+ raise HTTPException(+ status_code=403,+ detail={"error": "permission_denied", "required": list(required)},+ )+ return user++ return _dep++def require_api_key_permission(perm: str):"""Like ``require_permission``, but for the /v1 (API-key / forwarded-JWT)surface: resolves identity via ``get_api_key_user_async`` instead of@@ -459,6 +491,51 @@ def require_api_key_permission(perm: str):return _dep+def require_api_key_any_permission(required_perms: Sequence[str]):+ """Require at least one permission for an API-key or forwarded-JWT request."""+ required = tuple(required_perms)+ if not required:+ raise ValueError("require_api_key_any_permission requires at least one permission")++ from services.rbac_service import is_rbac_enforced++ async def _dep(+ request: Request,+ user: User = Depends(get_api_key_user_async),+ rbac=Depends(get_rbac_service),+ ) -> User:+ if not is_rbac_enforced():+ return user+ db_user_id = user.db_user_id or user.user_id+ role_override = getattr(request.state, "api_key_role_ids", None)+ perms = await rbac.get_user_permissions(db_user_id, role_override=role_override)+ if not any(perm in perms for perm in required):+ await rbac.audit_denied(db_user_id, "|".join(required))+ raise HTTPException(+ status_code=403,+ detail={"error": "permission_denied", "required": list(required)},+ )+ return user++ return _dep+++async def has_effective_permission(+ request: Request,+ user: User,+ rbac,+ perm: str,+) -> bool:+ """Check a permission using the same RBAC and API-key override semantics as route gates."""+ from services.rbac_service import is_rbac_enforced++ if not is_rbac_enforced():+ return True+ user_id = user.db_user_id or user.user_id+ role_override = getattr(request.state, "api_key_role_ids", None)+ return await rbac.has_permission(user_id, perm, role_override=role_override)++def require_all_permissions(required_perms: Sequence[str]):"""FastAPI dependency factory enforcing all listed permissions."""required = tuple(required_perms)diff --git a/src/models/processors.py b/src/models/processors.pyindex 0db7e77cc..af46f5e4a 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -5,6 +5,7 @@ import timefrom typing import TYPE_CHECKING, Any, Literalfrom config.settings import clients, get_embedding_model, get_index_name, get_openrag_config+from session_manager import AnonymousUserfrom utils.document_processing import (extract_relevant,process_text_file,@@ -40,6 +41,26 @@ def _verification_client(fallback_client):return clients.opensearch if clients.opensearch is not None else fallback_client+def resolve_shared_owner_fields(+ user_id: str | None,+ owner_name: str | None,+ owner_email: str | None,+ shared: bool,+) -> tuple[str | None, str | None, str | None]:+ """Return (owner, owner_name, owner_email) for indexing.++ When shared=True, owner is None so the indexed chunk omits the owner field+ entirely, triggering the OpenSearch DLS must_not-exists-owner clause that+ makes the document visible to all users in the instance. owner_name and+ owner_email are set to AnonymousUser values, matching how default/sample+ documents are loaded.+ """+ if shared:+ _anon = AnonymousUser()+ return None, _anon.name, _anon.email+ return user_id, owner_name, owner_email++class TaskProcessor:"""Base class for task processors with shared processing logic"""@@ -186,13 +207,17 @@ class TaskProcessor:filename: str,opensearch_client,owner_user_id: str | None = None,+ shared: bool = False,) -> None:"""Delete all chunks of a document with the given filename from OpenSearch."""from config.settings import clients, get_index_namefrom utils.opensearch_delete import collect_visible_document_ids, delete_document_ids- from utils.opensearch_queries import build_owned_filename_query+ from utils.opensearch_queries import (+ build_owned_filename_query,+ build_replace_filename_query,+ )try:write_client = clients.opensearch@@ -214,11 +239,18 @@ class TaskProcessor:filename=filename,)return++ # When shared=True the document being replaced may have previously+ # been ingested without an owner field (also shared), so the normal+ # owner-scoped query would miss those chunks. Use a broader query+ # that covers both owned and ownerless chunks for this filename.+ build_query = build_replace_filename_query if shared else build_owned_filename_query+for candidate in candidate_filenames:document_ids = await collect_visible_document_ids(opensearch_client,index=get_index_name(),- query=build_owned_filename_query(candidate, owner_user_id),+ query=build_query(candidate, owner_user_id),)deleted_count += await delete_document_ids(write_client,@@ -239,6 +271,7 @@ class TaskProcessor:opensearch_client,owner_user_id: str,keep_filenames: list[str] | None = None,+ shared: bool = False,) -> int:"""Delete indexed chunks for a connector file by its STABLE id.@@ -259,6 +292,11 @@ class TaskProcessor:if write_client is None:raise RuntimeError("Backend OpenSearch write client is unavailable")+ owner_filter = (+ {"bool": {"must_not": {"exists": {"field": "owner"}}}}+ if shared+ else {"term": {"owner": owner_user_id}}+ )query: dict[str, Any] = {"bool": {"filter": [@@ -271,7 +309,7 @@ class TaskProcessor:"minimum_should_match": 1,}},- {"term": {"owner": owner_user_id}},+ owner_filter,]}}@@ -315,6 +353,7 @@ class TaskProcessor:connector_file_id: str | None = None,ocr: bool | None = None,picture_descriptions: bool | None = None,+ shared: bool = False,):"""Standard processing pipeline for non-Langflow processors:@@ -482,9 +521,11 @@ class TaskProcessor:error=str(e),)- # Owner is always the authenticated uploading/syncing user. Upstream ACL- # owners/authors only contribute read access through allowed principals.- owner = owner_user_id+ # Owner is always the authenticated uploading/syncing user unless shared=True,+ # in which case owner fields are omitted so DLS makes the doc visible to all users.+ owner, owner_name, owner_email = resolve_shared_owner_fields(+ owner_user_id, owner_name, owner_email, shared+ )if acl:allowed_users = acl.allowed_users or []allowed_groups = acl.allowed_groups or []@@ -718,6 +759,7 @@ class ConnectorFileProcessor(TaskProcessor):ingest_settings: dict[str, Any] | None = None,replace_duplicates: bool = False,connector_type: str | None = None,+ shared: bool = False,):super().__init__(document_service=document_service,@@ -734,6 +776,7 @@ class ConnectorFileProcessor(TaskProcessor):self.ingest_settings = ingest_settingsself.replace_duplicates = replace_duplicatesself.connector_type = connector_type+ self.shared = sharedasync def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a connector file using unified methods"""@@ -813,7 +856,7 @@ class ConnectorFileProcessor(TaskProcessor):))deleted_chunks = await self._delete_connector_chunks(- file_id, opensearch_client, self.user_id+ file_id, opensearch_client, self.user_id, shared=self.shared)logger.warning(@@ -876,6 +919,7 @@ class ConnectorFileProcessor(TaskProcessor):opensearch_client,self.user_id,keep_filenames=get_filename_aliases(file_task.filename),+ shared=self.shared,)> 0)@@ -896,6 +940,7 @@ class ConnectorFileProcessor(TaskProcessor):file_task.filename,opensearch_client,owner_user_id=self.user_id,+ shared=self.shared,)# Create temporary file from document content@@ -983,15 +1028,20 @@ class ConnectorFileProcessor(TaskProcessor):{}, connector_tweak_settings)+ effective_owner, effective_owner_name, effective_owner_email = (+ resolve_shared_owner_fields(+ self.user_id, self.owner_name, self.owner_email, self.shared+ )+ )result = await self.connector_service.langflow_service.upload_and_ingest_file(file_tuple=file_tuple,session_id=None,tweaks=tweaks,settings=self.ingest_settings,jwt_token=self.jwt_token,- owner=self.user_id,- owner_name=self.owner_name,- owner_email=self.owner_email,+ owner=effective_owner,+ owner_name=effective_owner_name,+ owner_email=effective_owner_email,connector_type=connector_type,docling_polling_service=self.connector_service.task_service.docling_polling_serviceif self.connector_service.task_service@@ -1051,6 +1101,7 @@ class ConnectorFileProcessor(TaskProcessor):connector_type=connector_type,acl=document.acl,connector_file_id=document.id,+ shared=self.shared,**standard_kwargs,)diff --git a/src/services/document_index_writer.py b/src/services/document_index_writer.pyindex f3374386f..7a43a0fe9 100644--- a/src/services/document_index_writer.py+++ b/src/services/document_index_writer.py@@ -208,7 +208,6 @@ class DocumentIndexWriter:else metadata.get("file_size"),"connector_type": context.connector_type or metadata.get("connector_type") or "local","source_url": context.source_url or metadata.get("source_url") or "",- "owner": context.owner,"allowed_users": list(context.allowed_users),"allowed_groups": list(context.allowed_groups),"allowed_principals": unique_acl_principals(context.allowed_principals),@@ -219,6 +218,8 @@ class DocumentIndexWriter:"metadata": metadata.get("metadata", {}),}+ if context.owner is not None:+ doc["owner"] = context.ownerif context.owner_name is not None:doc["owner_name"] = context.owner_nameif context.owner_email is not None:diff --git a/src/services/langflow_file_service.py b/src/services/langflow_file_service.pyindex 799aeae79..a66100367 100644--- a/src/services/langflow_file_service.py+++ b/src/services/langflow_file_service.py@@ -377,7 +377,7 @@ class LangflowFileService:# Pass metadata via tweaks to OpenSearch componentmetadata_tweaks = []- if owner or owner is None:+ if owner:metadata_tweaks.append({"key": "owner", "value": owner})if owner_name:metadata_tweaks.append({"key": "owner_name", "value": owner_name})@@ -422,9 +422,9 @@ class LangflowFileService:headers = {"X-Langflow-Global-Var-JWT": str(jwt_token or ""),- "X-Langflow-Global-Var-OWNER": str(owner),- "X-Langflow-Global-Var-OWNER_NAME": str(owner_name),- "X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email),+ "X-Langflow-Global-Var-OWNER": owner or "",+ "X-Langflow-Global-Var-OWNER_NAME": owner_name or "",+ "X-Langflow-Global-Var-OWNER_EMAIL": owner_email or "","X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),"X-Langflow-Global-Var-MIMETYPE": mimetype,"X-Langflow-Global-Var-FILESIZE": str(file_size_bytes),@@ -595,9 +595,9 @@ class LangflowFileService:resolved_document_id = hash_id(io.BytesIO(docs_url.encode("utf-8")))headers = {"X-Langflow-Global-Var-JWT": str(jwt_token or ""),- "X-Langflow-Global-Var-OWNER": str(owner),- "X-Langflow-Global-Var-OWNER_NAME": str(owner_name),- "X-Langflow-Global-Var-OWNER_EMAIL": str(owner_email),+ "X-Langflow-Global-Var-OWNER": owner or "",+ "X-Langflow-Global-Var-OWNER_NAME": owner_name or "",+ "X-Langflow-Global-Var-OWNER_EMAIL": owner_email or "","X-Langflow-Global-Var-CONNECTOR_TYPE": str(connector_type),"X-Langflow-Global-Var-SELECTED_EMBEDDING_MODEL": str(embedding_model),"X-Langflow-Global-Var-DOCUMENT_ID": resolved_document_id,diff --git a/src/utils/opensearch_queries.py b/src/utils/opensearch_queries.pyindex a99d3227f..7e586151b 100644--- a/src/utils/opensearch_queries.py+++ b/src/utils/opensearch_queries.py@@ -43,3 +43,44 @@ def build_owned_filename_query(filename: str, owner: str) -> dict:]}}+++def build_anonymous_filename_query(filename: str) -> dict:+ """Build a query for ownerless chunks with a specific filename."""+ return {+ "bool": {+ "filter": [+ build_filename_query(filename),+ {"bool": {"must_not": {"exists": {"field": "owner"}}}},+ ]+ }+ }+++def build_replace_filename_query(filename: str, owner: str) -> dict:+ """Build a delete-scope query for replace_duplicates that covers both private+ and shared (ownerless) chunks with this filename.++ Matches chunks where filename matches AND (owner == current user OR owner+ field is absent). Combining both cases is necessary because the same+ filename may have been previously ingested as shared (no owner field) and+ is now being replaced by the same user. The owner-field branch protects+ against accidentally deleting documents owned by *other* users that are+ merely visible to the current user via allowed_users DLS.+ """+ return {+ "bool": {+ "filter": [+ build_filename_query(filename),+ {+ "bool": {+ "should": [+ {"term": {"owner": owner}},+ {"bool": {"must_not": {"exists": {"field": "owner"}}}},+ ],+ "minimum_should_match": 1,+ }+ },+ ]+ }+ }diff --git a/tests/integration/core/test_shared_flag_dls.py b/tests/integration/core/test_shared_flag_dls.pynew file mode 100644index 000000000..95a9f3cb9--- /dev/null+++ b/tests/integration/core/test_shared_flag_dls.py@@ -0,0 +1,228 @@+"""Integration tests: shared-flag DLS anonymous path in OpenSearch.++These tests verify that documents indexed without an owner field are visible+to all authenticated users via the existing must_not-exists-owner DLS clause,+and that documents WITH an owner field remain private to their owner.++Requires a live OpenSearch instance with DLS configured (OPENSEARCH_PASSWORD set).+"""++from uuid import uuid4++import pytest+from opensearchpy import AsyncOpenSearch+from opensearchpy._async.http_aiohttp import AIOHttpConnection++pytestmark = [+ pytest.mark.asyncio,+ pytest.mark.openrag_skip_app_onboard,+]+++def _build_admin_opensearch_client():+ from config.settings import (+ IBM_AUTH_ENABLED,+ OPENSEARCH_HOST,+ OPENSEARCH_PASSWORD,+ OPENSEARCH_PORT,+ OPENSEARCH_USERNAME,+ )++ if IBM_AUTH_ENABLED:+ pytest.skip("OSS JWT DLS is not used in IBM auth mode")+ if not OPENSEARCH_PASSWORD:+ pytest.skip("OPENSEARCH_PASSWORD is required for this DLS integration test")++ return AsyncOpenSearch(+ hosts=[{"host": OPENSEARCH_HOST, "port": OPENSEARCH_PORT}],+ connection_class=AIOHttpConnection,+ scheme="https",+ use_ssl=True,+ verify_certs=False,+ ssl_assert_fingerprint=None,+ http_auth=(OPENSEARCH_USERNAME, OPENSEARCH_PASSWORD),+ http_compress=True,+ )+++async def _search_visible_document_ids(opensearch_client, index_name: str) -> set[str]:+ response = await opensearch_client.search(+ index=index_name,+ body={+ "query": {"match_all": {}},+ "_source": ["document_id"],+ "size": 20,+ },+ )+ return {hit["_source"]["document_id"] for hit in response.get("hits", {}).get("hits", [])}+++async def test_ownerless_doc_visible_to_all_users():+ """A document indexed without an owner field is visible to all authenticated users.++ This is the core DLS mechanism used by shared=True ingestion.+ The must_not-exists-owner clause in securityconfig/roles.yml makes any+ chunk whose owner key is absent universally readable.+ """+ from config.settings import INDEX_BODY, clients+ from session_manager import SessionManager, User+ from utils.opensearch_utils import setup_opensearch_security++ admin_client = _build_admin_opensearch_client()+ try:+ is_reachable = await admin_client.ping()+ except Exception:+ is_reachable = False+ if not is_reachable:+ await admin_client.close()+ pytest.skip("OpenSearch is not reachable")++ index_name = f"documents_shared_flag_dls_{uuid4().hex}"+ user_a_id = f"user-a-{uuid4().hex}"+ user_a_email = f"{user_a_id}@example.com"+ user_b_id = f"user-b-{uuid4().hex}"+ user_b_email = f"{user_b_id}@example.com"++ try:+ await setup_opensearch_security(admin_client)+ await admin_client.indices.create(index=index_name, body=INDEX_BODY)++ await admin_client.bulk(+ body=[+ # Shared doc: no owner key → visible to everyone+ {"index": {"_index": index_name, "_id": "shared-doc"}},+ {+ "document_id": "shared-doc",+ "filename": "shared.pdf",+ "text": "Shared document visible to all users",+ "allowed_users": [],+ "allowed_groups": [],+ "allowed_principals": [],+ },+ # Private doc owned by user A → only visible to user A+ {"index": {"_index": index_name, "_id": "private-a-doc"}},+ {+ "document_id": "private-a-doc",+ "filename": "private-a.pdf",+ "text": "Private document owned by user A",+ "owner": user_a_id,+ "allowed_users": [],+ "allowed_groups": [],+ "allowed_principals": [],+ },+ # Private doc owned by user B → only visible to user B+ {"index": {"_index": index_name, "_id": "private-b-doc"}},+ {+ "document_id": "private-b-doc",+ "filename": "private-b.pdf",+ "text": "Private document owned by user B",+ "owner": user_b_id,+ "allowed_users": [],+ "allowed_groups": [],+ "allowed_principals": [],+ },+ ],+ refresh=True,+ )++ session_manager = SessionManager("test")++ user_a = User(user_id=user_a_id, email=user_a_email, name="User A")+ token_a = session_manager.create_opensearch_jwt_token(user_a, ttl_seconds=120)+ client_a = clients.create_user_opensearch_client(token_a)++ user_b = User(user_id=user_b_id, email=user_b_email, name="User B")+ token_b = session_manager.create_opensearch_jwt_token(user_b, ttl_seconds=120)+ client_b = clients.create_user_opensearch_client(token_b)++ try:+ visible_a = await _search_visible_document_ids(client_a, index_name)+ visible_b = await _search_visible_document_ids(client_b, index_name)++ # Shared doc visible to both users+ assert "shared-doc" in visible_a, "User A must see the shared (ownerless) document"+ assert "shared-doc" in visible_b, "User B must see the shared (ownerless) document"++ # Each user sees only their own private doc (not the other user's)+ assert "private-a-doc" in visible_a+ assert "private-a-doc" not in visible_b, "User B must NOT see User A's private doc"++ assert "private-b-doc" in visible_b+ assert "private-b-doc" not in visible_a, "User A must NOT see User B's private doc"++ finally:+ await client_a.close()+ await client_b.close()++ finally:+ await admin_client.indices.delete(index=index_name, ignore_unavailable=True)+ await admin_client.close()+++async def test_null_owner_field_does_not_trigger_anonymous_path():+ """A document indexed with owner=null is universally visible — same as an absent owner key.++ OpenSearch does not index null values for keyword fields without a null_value mapping.+ The exists filter therefore returns false for 'owner: null', so the+ must_not-exists-owner DLS clause fires and the document is visible to everyone.++ This documents that _build_chunk_document correctly omits the owner key when None,+ making intent explicit; both null and absent produce the same DLS result.+ """+ from config.settings import INDEX_BODY, clients+ from session_manager import SessionManager, User+ from utils.opensearch_utils import setup_opensearch_security++ admin_client = _build_admin_opensearch_client()+ try:+ is_reachable = await admin_client.ping()+ except Exception:+ is_reachable = False+ if not is_reachable:+ await admin_client.close()+ pytest.skip("OpenSearch is not reachable")++ index_name = f"documents_null_owner_dls_{uuid4().hex}"+ unrelated_user_id = f"unrelated-{uuid4().hex}"+ unrelated_email = f"{unrelated_user_id}@example.com"++ try:+ await setup_opensearch_security(admin_client)+ await admin_client.indices.create(index=index_name, body=INDEX_BODY)++ # Index a doc with owner: null explicitly (the broken serialization path)+ await admin_client.index(+ index=index_name,+ id="null-owner-doc",+ body={+ "document_id": "null-owner-doc",+ "filename": "null-owner.pdf",+ "text": "Document with owner field set to null",+ "owner": None,+ "allowed_users": [],+ "allowed_groups": [],+ "allowed_principals": [],+ },+ refresh=True,+ )++ session_manager = SessionManager("test")+ unrelated = User(user_id=unrelated_user_id, email=unrelated_email, name="Unrelated User")+ token = session_manager.create_opensearch_jwt_token(unrelated, ttl_seconds=120)+ client = clients.create_user_opensearch_client(token)++ try:+ visible = await _search_visible_document_ids(client, index_name)+ # owner=null is not indexed by OpenSearch (keyword, no null_value), so exists+ # returns false → must_not-exists-owner fires → doc is visible to everyone+ assert "null-owner-doc" in visible, (+ "A doc with owner=null must be universally visible: OpenSearch does not "+ "index null keyword values, so must_not-exists-owner fires just as it "+ "does for a doc with no owner key at all."+ )+ finally:+ await client.close()++ finally:+ await admin_client.indices.delete(index=index_name, ignore_unavailable=True)+ await admin_client.close()diff --git a/tests/unit/db/migrations/test_add_anonymous_delete_permission.py b/tests/unit/db/migrations/test_add_anonymous_delete_permission.pynew file mode 100644index 000000000..31bd6f505--- /dev/null+++ b/tests/unit/db/migrations/test_add_anonymous_delete_permission.py@@ -0,0 +1,97 @@+"""Tests for the anonymous document-delete permission migration."""++import importlib.util+from pathlib import Path++import sqlalchemy as sa++ROOT = Path(__file__).resolve().parent.parent.parent.parent.parent+MIGRATION_PATH = ROOT / "alembic" / "versions" / "0007_add_knowledge_delete_anonymous.py"+++def _load_migration():+ spec = importlib.util.spec_from_file_location("migration_0007", MIGRATION_PATH)+ assert spec is not None and spec.loader is not None+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def test_upgrade_is_idempotent_and_preserves_delete_any(monkeypatch):+ engine = sa.create_engine("sqlite:///:memory:")+ metadata = sa.MetaData()+ roles = sa.Table(+ "roles",+ metadata,+ sa.Column("id", sa.String, primary_key=True),+ sa.Column("name", sa.String, unique=True, nullable=False),+ )+ permissions = sa.Table(+ "permissions",+ metadata,+ sa.Column("id", sa.String, primary_key=True),+ sa.Column("name", sa.String, unique=True, nullable=False),+ sa.Column("resource", sa.String, nullable=False),+ sa.Column("action", sa.String, nullable=False),+ sa.Column("description", sa.String),+ )+ role_permissions = sa.Table(+ "role_permissions",+ metadata,+ sa.Column("role_id", sa.String, primary_key=True),+ sa.Column("permission_id", sa.String, primary_key=True),+ )+ metadata.create_all(engine)++ with engine.begin() as connection:+ connection.execute(+ roles.insert(),+ [+ {"id": "role-admin", "name": "admin"},+ {"id": "role-custom", "name": "custom"},+ ],+ )+ connection.execute(+ permissions.insert(),+ {+ "id": "perm-delete-any",+ "name": "knowledge:delete:any",+ "resource": "knowledge",+ "action": "delete:any",+ "description": "Delete any document",+ },+ )+ connection.execute(+ role_permissions.insert(),+ {"role_id": "role-custom", "permission_id": "perm-delete-any"},+ )++ migration = _load_migration()+ monkeypatch.setattr(migration.op, "get_bind", lambda: connection)+ migration.upgrade()+ migration.upgrade()++ permission_rows = connection.execute(+ sa.text(+ "SELECT id, name FROM permissions "+ "WHERE name IN ('knowledge:delete:any', 'knowledge:delete:anonymous') "+ "ORDER BY name"+ )+ ).fetchall()+ assert [row.name for row in permission_rows] == [+ "knowledge:delete:anonymous",+ "knowledge:delete:any",+ ]++ anonymous_id = next(+ row.id for row in permission_rows if row.name == "knowledge:delete:anonymous"+ )+ assignments = connection.execute(+ sa.text(+ "SELECT role_id, permission_id FROM role_permissions "+ "ORDER BY role_id, permission_id"+ )+ ).fetchall()+ assert ("role-admin", anonymous_id) in assignments+ assert ("role-custom", "perm-delete-any") in assignments+ assert len([row for row in assignments if row == ("role-admin", anonymous_id)]) == 1diff --git a/tests/unit/db/test_rbac_seed_idempotency.py b/tests/unit/db/test_rbac_seed_idempotency.pyindex 41749d1e9..bb8c4d7a9 100644--- a/tests/unit/db/test_rbac_seed_idempotency.py+++ b/tests/unit/db/test_rbac_seed_idempotency.py@@ -48,6 +48,9 @@ async def test_seed_creates_expected_rows(session):assert len(perms) == len(PERMISSIONS)assert {r.name for r in roles} == {n for _, n, _ in BUILTIN_ROLES}+ perm_id_by_name = {p.name: p.id for p in perms}+ assert "knowledge:delete:any" in perm_id_by_name+ assert "knowledge:delete:anonymous" in perm_id_by_name# admin gets every permissionadmin_role = next(r for r in roles if r.name == "admin")@@ -62,6 +65,23 @@ async def test_seed_creates_expected_rows(session):.all()}assert len(admin_perm_ids) == len(PERMISSIONS)+ assert perm_id_by_name["knowledge:delete:any"] in admin_perm_ids+ assert perm_id_by_name["knowledge:delete:anonymous"] in admin_perm_ids++ for role_name in ("developer", "user"):+ role = next(r for r in roles if r.name == role_name)+ role_perm_ids = {+ rp.permission_id+ for rp in (+ await session.execute(+ select(RolePermission).where(RolePermission.role_id == role.id)+ )+ )+ .scalars()+ .all()+ }+ assert perm_id_by_name["knowledge:delete:own"] in role_perm_ids+ assert perm_id_by_name["knowledge:delete:anonymous"] not in role_perm_ids@pytest.mark.asynciodiff --git a/tests/unit/dependencies/test_rbac_kill_switch.py b/tests/unit/dependencies/test_rbac_kill_switch.pyindex 888da9ca3..d28a81c95 100644--- a/tests/unit/dependencies/test_rbac_kill_switch.py+++ b/tests/unit/dependencies/test_rbac_kill_switch.py@@ -109,12 +109,22 @@ async def app(monkeypatch):# Mount a minimal endpoint behind a real require_permission gate so the# kill switch can be exercised without depending on any specific router.- from dependencies import require_permission+ from dependencies import require_any_permission, require_permission++ require_anonymous_delete_permission = require_any_permission(+ ("knowledge:delete:anonymous", "users:delete")+ )@fastapi_app.get("/admin/users")async def _gated(_=Depends(require_permission("users:list"))):return []+ @fastapi_app.delete("/anonymous-documents")+ async def _any_gated(+ _=Depends(require_anonymous_delete_permission),+ ):+ return []+yield fastapi_app, SessionLocal, rbac, personasawait engine.dispose()@@ -139,6 +149,20 @@ async def test_kill_switch_bypasses_require_permission(app, monkeypatch):assert r.status_code == 200+async def test_kill_switch_bypasses_require_any_permission(app, monkeypatch):+ fastapi_app, _, _, _ = app+ transport = httpx.ASGITransport(app=fastapi_app)+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:+ monkeypatch.setenv("OPENRAG_RBAC_ENFORCE", "true")+ r = await c.delete("/anonymous-documents", headers={"X-Test-Persona": "user"})+ assert r.status_code == 403++ monkeypatch.setenv("OPENRAG_RBAC_ENFORCE", "false")+ r = await c.delete("/anonymous-documents", headers={"X-Test-Persona": "user"})+ assert r.status_code == 200++@pytest.mark.asyncioasync def test_kill_switch_bypasses_assert_owner_or_perm(monkeypatch):"""The shared helper used by `:own` endpoints must also bypass."""diff --git a/tests/unit/dependencies/test_require_api_key_permission.py b/tests/unit/dependencies/test_require_api_key_permission.pyindex e9b4a6258..e8c651016 100644--- a/tests/unit/dependencies/test_require_api_key_permission.py+++ b/tests/unit/dependencies/test_require_api_key_permission.py@@ -15,6 +15,7 @@ import httpximport pytestimport pytest_asynciofrom fastapi import Depends, FastAPI, Request+from sqlalchemy import selectfrom sqlalchemy.ext.asyncio import async_sessionmaker, create_async_enginefrom sqlmodel import SQLModel@@ -26,6 +27,7 @@ if str(SRC) not in sys.path:import db.models # noqa: E402,F401from api.v1.documents import delete_document_endpoint # noqa: E402from api.v1.knowledge_filters import search_endpoint as kf_search_endpoint # noqa: E402+from db.models import Permission, Role, RolePermission # noqa: E402from db.repositories import RoleRepo # noqa: E402from db.seed import seed_roles_and_permissions # noqa: E402from dependencies import ( # noqa: E402@@ -33,12 +35,17 @@ from dependencies import ( # noqa: E402get_knowledge_filter_service,get_rbac_service,get_session_manager,+ require_api_key_any_permission,require_api_key_permission,)from services.rbac_service import RBACService # noqa: E402from services.user_service import ensure_user_row # noqa: E402from session_manager import User # noqa: E402+require_api_key_delete_permission = require_api_key_any_permission(+ ("knowledge:delete:anonymous", "users:delete")+)+@pytest_asyncio.fixtureasync def app(monkeypatch):@@ -78,6 +85,40 @@ async def app(monkeypatch):personas["norole"] = User(user_id=norole_row.id, email="norole@x", name="norole", provider="ibm_ams")++ delete_any_row = await ensure_user_row(+ s,+ User(+ user_id="delete-any-sub",+ email="delete-any@x",+ name="delete-any",+ provider="ibm_ams",+ ),+ )+ await role_repo.revoke_role(delete_any_row.id, user_role.id)+ delete_any_role = Role(+ id="role-api-delete-any-only",+ name="api-delete-any-only",+ description="Legacy unrestricted delete permission only",+ )+ s.add(delete_any_role)+ await s.flush()+ delete_any_permission = (+ await s.execute(select(Permission).where(Permission.name == "knowledge:delete:any"))+ ).scalar_one()+ s.add(+ RolePermission(+ role_id=delete_any_role.id,+ permission_id=delete_any_permission.id,+ )+ )+ await role_repo.assign_role(delete_any_row.id, delete_any_role.id)+ personas["delete_any"] = User(+ user_id=delete_any_row.id,+ email="delete-any@x",+ name="delete-any",+ provider="ibm_ams",+ )await s.commit()rbac = RBACService(SessionLocal)@@ -104,6 +145,10 @@ async def app(monkeypatch):async def _probe_kf_read(user=Depends(require_api_key_permission("kf:read"))):return {"user_id": user.user_id}+ @fastapi_app.get("/probe/delete-document")+ async def _probe_delete_document(user=Depends(require_api_key_delete_permission)):+ return {"user_id": user.user_id}+# A real gated /v1 handler. The gate runs as a dependency *before* the# handler body, so a denied request never reaches the delete core (no# service overrides needed for the 403 path).@@ -130,6 +175,15 @@ async def test_kill_switch_off_bypasses(app, monkeypatch):assert r.status_code == 200+async def test_kill_switch_off_bypasses_api_key_any_permission(app, monkeypatch):+ monkeypatch.setenv("OPENRAG_RBAC_ENFORCE", "false")+ fastapi_app, _ = app+ async with _client(fastapi_app) as c:+ r = await c.get("/probe/delete-document", headers={"X-Test-Persona": "user"})+ assert r.status_code == 200++@pytest.mark.asyncioasync def test_admin_passes_when_enforced(app, monkeypatch):monkeypatch.setenv("OPENRAG_RBAC_ENFORCE", "true")@@ -188,9 +242,7 @@ async def test_real_v1_kf_search_gated_on_kf_read(app, monkeypatch):@pytest.mark.asyncioasync def test_real_v1_delete_blocks_viewer(app, monkeypatch):- """End-to-end wiring: DELETE /v1/documents is gated on knowledge:delete:own.- A viewer lacks it, so the real handler returns 403 from the gate before the- body runs."""+ """The v1 delete endpoint accepts own or anonymous scope and rejects a viewer."""monkeypatch.setenv("OPENRAG_RBAC_ENFORCE", "true")fastapi_app, _ = appasync with _client(fastapi_app) as c:@@ -201,4 +253,25 @@ async def test_real_v1_delete_blocks_viewer(app, monkeypatch):headers={"X-Test-Persona": "viewer"},)assert r.status_code == 403- assert r.json()["detail"]["required"] == "knowledge:delete:own"+ assert r.json()["detail"]["required"] == [+ "knowledge:delete:own",+ "knowledge:delete:anonymous",+ ]+++async def test_real_v1_delete_blocks_delete_any_only_role(app, monkeypatch):+ monkeypatch.setenv("OPENRAG_RBAC_ENFORCE", "true")+ fastapi_app, _ = app+ async with _client(fastapi_app) as c:+ r = await c.request(+ "DELETE",+ "/v1/documents",+ json={"filename": "x.txt"},+ headers={"X-Test-Persona": "delete_any"},+ )+ assert r.status_code == 403+ assert r.json()["detail"]["required"] == [+ "knowledge:delete:own",+ "knowledge:delete:anonymous",+ ]diff --git a/tests/unit/dependencies/test_require_permission.py b/tests/unit/dependencies/test_require_permission.pyindex 719797f14..39600f359 100644--- a/tests/unit/dependencies/test_require_permission.py+++ b/tests/unit/dependencies/test_require_permission.py@@ -15,6 +15,7 @@ import pytestimport pytest_asynciofrom fastapi import Depends, FastAPI, Requestfrom fastapi.responses import JSONResponse+from sqlalchemy import selectfrom sqlalchemy.ext.asyncio import async_sessionmaker, create_async_enginefrom sqlmodel import SQLModel@@ -24,6 +25,11 @@ if str(SRC) not in sys.path:sys.path.insert(0, str(SRC))import db.models # noqa: E402,F401+from db.models import ( # noqa: E402+ Permission,+ Role,+ RolePermission,+)from db.models import User as UserRow # noqa: E402from db.repositories import RoleRepo # noqa: E402from db.seed import seed_roles_and_permissions # noqa: E402@@ -33,6 +39,7 @@ from dependencies import ( # noqa: E402get_current_user,get_rbac_service,require_all_permissions,+ require_any_permission,require_permission,)from services.rbac_service import RBACService # noqa: E402@@ -41,6 +48,9 @@ from session_manager import User # noqa: E402# Map role-name -> User dataclass we hand to overridesPERSONAS: dict[str, User] = {}+require_document_delete_permission = require_any_permission(+ ("knowledge:delete:own", "knowledge:delete:anonymous")+)@pytest_asyncio.fixture@@ -88,6 +98,34 @@ async def app(monkeypatch):await role_repo.revoke_role(viewer_db.id, user_role.id)await role_repo.assign_role(viewer_db.id, viewer_role.id)+ delete_any_db = await ensure_user_row(+ s,+ User(+ user_id="delete-any-sub",+ email="[email protected]",+ name="Delete Any",+ provider="google",+ ),+ )+ await role_repo.revoke_role(delete_any_db.id, user_role.id)+ delete_any_role = Role(+ id="role-delete-any-only",+ name="delete-any-only",+ description="Legacy unrestricted delete permission only",+ )+ s.add(delete_any_role)+ await s.flush()+ delete_any_permission = (+ await s.execute(select(Permission).where(Permission.name == "knowledge:delete:any"))+ ).scalar_one()+ s.add(+ RolePermission(+ role_id=delete_any_role.id,+ permission_id=delete_any_permission.id,+ )+ )+ await role_repo.assign_role(delete_any_db.id, delete_any_role.id)+aliased_db = UserRow(id="db-aliased-user",oauth_provider="google",@@ -113,6 +151,12 @@ async def app(monkeypatch):PERSONAS["viewer"] = User(user_id=viewer_db.id, email="[email protected]", name="V", provider="google")+ PERSONAS["delete_any"] = User(+ user_id=delete_any_db.id,+ email="[email protected]",+ name="Delete Any",+ provider="google",+ )PERSONAS["aliased"] = User(user_id="oauth-aliased-sub",email="[email protected]",@@ -158,6 +202,12 @@ async def app(monkeypatch):):return JSONResponse({"user_id": user.user_id})+ @app.post("/test/delete-document")+ async def delete_document_gate(+ user=Depends(require_document_delete_permission),+ ):+ return JSONResponse({"user_id": user.user_id})+@app.post("/test/whoami")async def whoami(user=Depends(require_permission("chat:use"))):return JSONResponse({"user_id": user.user_id, "db_user_id": user.db_user_id})@@ -198,6 +248,11 @@ async def app(monkeypatch):# upload-context is both ingestion and chat behavior("user", "/test/upload-context", 200),("viewer", "/test/upload-context", 403),+ # delete accepts either own or anonymous-document scope+ ("admin", "/test/delete-document", 200),+ ("user", "/test/delete-document", 200),+ ("viewer", "/test/delete-document", 403),+ ("delete_any", "/test/delete-document", 403),],)async def test_permission_enforcement(app, persona, perm_path, expected):@@ -228,6 +283,18 @@ async def test_all_permissions_response_lists_required_permissions(app):assert r.json()["detail"]["required"] == ["knowledge:upload", "chat:use"]+async def test_any_permission_response_lists_alternatives(app):+ transport = httpx.ASGITransport(app=app)+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:+ r = await c.post("/test/delete-document", headers={"X-Test-Persona": "viewer"})+ assert r.status_code == 403+ assert r.json()["detail"]["required"] == [+ "knowledge:delete:own",+ "knowledge:delete:anonymous",+ ]++@pytest.mark.asyncioasync def test_permission_check_uses_resolved_db_user_id(app):transport = httpx.ASGITransport(app=app)diff --git a/tests/unit/test_document_delete_by_id.py b/tests/unit/test_document_delete_by_id.pyindex aa6983258..c8844434d 100644--- a/tests/unit/test_document_delete_by_id.py+++ b/tests/unit/test_document_delete_by_id.py@@ -94,6 +94,136 @@ async def test_delete_documents_by_filename_denies_visible_non_owner(monkeypatchassert opensearch_client.delete_calls == []+async def test_delete_documents_by_filename_deletes_ownerless_with_anonymous_permission(+ monkeypatch,+):+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents")+ opensearch_client = FakeOpenSearchClient(+ visible_hits=[{"_id": "shared-chunk", "_source": {}}],+ )+ backend_opensearch_client = FakeOpenSearchClient(+ owned_hits=[+ {"_id": "shared-chunk-1", "_source": {}},+ {"_id": "shared-chunk-2", "_source": {}},+ ]+ )+ monkeypatch.setattr("config.settings.clients.opensearch", backend_opensearch_client)++ payload, status_code = await delete_documents_by_filename_core(+ filename="shared.pdf",+ session_manager=FakeSessionManager(opensearch_client),+ user_id="user-1",+ jwt_token="jwt-token",+ can_delete_own=False,+ can_delete_anonymous=True,+ )++ assert status_code == 200+ assert payload["success"] is True+ assert payload["deleted_chunks"] == 2+ assert opensearch_client.search_calls == []+ assert backend_opensearch_client.search_calls[0]["body"]["query"] == {+ "bool": {+ "filter": [+ {"term": {"filename": "shared.pdf"}},+ {"bool": {"must_not": {"exists": {"field": "owner"}}}},+ ]+ }+ }+ assert backend_opensearch_client.delete_calls == [+ {"index": "documents", "id": "shared-chunk-1", "refresh": True},+ {"index": "documents", "id": "shared-chunk-2", "refresh": True},+ ]+++async def test_delete_documents_by_filename_combines_owned_and_anonymous_scopes(monkeypatch):+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents")+ opensearch_client = FakeOpenSearchClient()+ backend_opensearch_client = FakeOpenSearchClient(+ owned_hits=[+ {"_id": "owned-chunk", "_source": {"owner": "user-1"}},+ {"_id": "anonymous-chunk", "_source": {}},+ ]+ )+ monkeypatch.setattr("config.settings.clients.opensearch", backend_opensearch_client)++ payload, status_code = await delete_documents_by_filename_core(+ filename="shared.pdf",+ session_manager=FakeSessionManager(opensearch_client),+ user_id="user-1",+ jwt_token="jwt-token",+ can_delete_own=True,+ can_delete_anonymous=True,+ )++ assert status_code == 200+ assert payload["deleted_chunks"] == 2+ assert backend_opensearch_client.search_calls[0]["body"]["query"] == {+ "bool": {+ "filter": [+ {"term": {"filename": "shared.pdf"}},+ {+ "bool": {+ "should": [+ {"term": {"owner": "user-1"}},+ {"bool": {"must_not": {"exists": {"field": "owner"}}}},+ ],+ "minimum_should_match": 1,+ }+ },+ ]+ }+ }+++async def test_delete_documents_by_filename_denies_ownerless_without_anonymous_permission(+ monkeypatch,+):+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents")+ opensearch_client = FakeOpenSearchClient(+ owned_hits=[],+ visible_hits=[{"_id": "shared-chunk", "_source": {}}],+ )++ payload, status_code = await delete_documents_by_filename_core(+ filename="shared.pdf",+ session_manager=FakeSessionManager(opensearch_client),+ user_id="user-1",+ jwt_token="jwt-token",+ can_delete_own=True,+ can_delete_anonymous=False,+ )++ assert status_code == 403+ assert payload["success"] is False+ assert payload["deleted_chunks"] == 0+ assert "only the document owner" in payload["error"]+ assert opensearch_client.delete_calls == []+++async def test_delete_documents_by_filename_denies_when_no_delete_scope(monkeypatch):+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents")+ opensearch_client = FakeOpenSearchClient()++ payload, status_code = await delete_documents_by_filename_core(+ filename="shared.pdf",+ session_manager=FakeSessionManager(opensearch_client),+ user_id="user-1",+ jwt_token="jwt-token",+ can_delete_own=False,+ can_delete_anonymous=False,+ )++ assert status_code == 403+ assert payload["success"] is False+ assert "insufficient permissions" in payload["error"]+ assert opensearch_client.search_calls == []++@pytest.mark.asyncioasync def test_delete_documents_by_filename_returns_404_when_missing(monkeypatch):monkeypatch.setattr("config.settings.get_index_name", lambda: "documents")diff --git a/tests/unit/test_shared_flag.py b/tests/unit/test_shared_flag.pynew file mode 100644index 000000000..8fc7f1c80--- /dev/null+++ b/tests/unit/test_shared_flag.py@@ -0,0 +1,237 @@+"""Unit tests for the shared COS ingestion flag."""++from typing import Any++import pytest++from models.processors import resolve_shared_owner_fields+from services.document_index_writer import (+ DocumentIndexChunk,+ DocumentIndexContext,+ DocumentIndexWriter,+)++# ---------------------------------------------------------------------------+# resolve_shared_owner_fields+# ---------------------------------------------------------------------------+++def test_resolve_shared_owner_fields_private():+ result = resolve_shared_owner_fields("user-1", "Alice", "[email protected]", shared=False)+ assert result == ("user-1", "Alice", "[email protected]")+++def test_resolve_shared_owner_fields_shared():+ result = resolve_shared_owner_fields("user-1", "Alice", "[email protected]", shared=True)+ assert result == (None, "Anonymous User", "anonymous@localhost")+++def test_resolve_shared_owner_fields_shared_none_inputs():+ result = resolve_shared_owner_fields(None, None, None, shared=True)+ assert result == (None, "Anonymous User", "anonymous@localhost")+++def test_resolve_shared_owner_fields_private_none_inputs():+ result = resolve_shared_owner_fields(None, None, None, shared=False)+ assert result == (None, None, None)+++# ---------------------------------------------------------------------------+# DocumentIndexWriter._build_chunk_document+# ---------------------------------------------------------------------------+++def _make_context(**kwargs):+ defaults: dict[str, Any] = dict(+ document_id="doc-1",+ filename="test.pdf",+ mimetype="application/pdf",+ embedding_model="test-model",+ file_size=None,+ allowed_users=[],+ allowed_groups=[],+ allowed_principals=[],+ allowed_principal_labels=[],+ is_sample_data=False,+ )+ defaults.update(kwargs)+ return DocumentIndexContext(**defaults)+++def _make_chunk():+ return DocumentIndexChunk(+ chunk_id="doc-1_0",+ text="hello world",+ vector=[0.1, 0.2, 0.3],+ page=1,+ )+++def test_build_chunk_document_owner_present_when_set():+ writer = DocumentIndexWriter()+ context = _make_context(owner="user-1", owner_name="Alice", owner_email="[email protected]")+ chunk = _make_chunk()+ doc = writer._build_chunk_document(+ context=context, chunk=chunk, embedding_field="vector", indexed_time="2026-01-01T00:00:00"+ )+ assert doc["owner"] == "user-1"+ assert doc["owner_name"] == "Alice"+ assert doc["owner_email"] == "[email protected]"+++def test_build_chunk_document_omits_owner_key_when_none():+ """Critical DLS test: owner key must be absent, not null, for must_not-exists-owner clause."""+ writer = DocumentIndexWriter()+ context = _make_context(owner=None, owner_name=None, owner_email=None)+ chunk = _make_chunk()+ doc = writer._build_chunk_document(+ context=context, chunk=chunk, embedding_field="vector", indexed_time="2026-01-01T00:00:00"+ )+ assert "owner" not in doc+ assert "owner_name" not in doc+ assert "owner_email" not in doc+++def test_build_chunk_document_shared_has_anonymous_metadata():+ """Shared docs: owner key absent for DLS, owner_name/email set to anonymous values."""+ writer = DocumentIndexWriter()+ context = _make_context(+ owner=None, owner_name="Anonymous User", owner_email="anonymous@localhost"+ )+ chunk = _make_chunk()+ doc = writer._build_chunk_document(+ context=context, chunk=chunk, embedding_field="vector", indexed_time="2026-01-01T00:00:00"+ )+ assert "owner" not in doc+ assert doc["owner_name"] == "Anonymous User"+ assert doc["owner_email"] == "anonymous@localhost"+++def test_build_chunk_document_allowed_users_always_present():+ """allowed_users/groups are always written (DLS lookup requires the field to exist)."""+ writer = DocumentIndexWriter()+ context = _make_context(owner=None)+ chunk = _make_chunk()+ doc = writer._build_chunk_document(+ context=context, chunk=chunk, embedding_field="vector", indexed_time="2026-01-01T00:00:00"+ )+ assert "allowed_users" in doc+ assert "allowed_groups" in doc+++# ---------------------------------------------------------------------------+# build_replace_filename_query+# ---------------------------------------------------------------------------+++def test_build_replace_filename_query_structure():+ """Must match filename AND (owner == user OR must_not-exists-owner)."""+ from utils.opensearch_queries import build_replace_filename_query++ q = build_replace_filename_query("report.pdf", "user-1")+ assert q["bool"]["filter"][0] == {"term": {"filename": "report.pdf"}}+ should = q["bool"]["filter"][1]["bool"]["should"]+ assert {"term": {"owner": "user-1"}} in should+ assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in should+ assert q["bool"]["filter"][1]["bool"]["minimum_should_match"] == 1+++def test_build_replace_filename_query_differs_from_owned_query():+ """replace query is broader than the owner-only query."""+ from utils.opensearch_queries import build_owned_filename_query, build_replace_filename_query++ owned = build_owned_filename_query("f.pdf", "u")+ replace = build_replace_filename_query("f.pdf", "u")+ # owned has a single term filter; replace has a bool/should+ assert owned != replace+++# ---------------------------------------------------------------------------+# ConnectorSyncBody Pydantic model+# ---------------------------------------------------------------------------+++def test_connector_sync_body_defaults_shared_false():+ from api.connectors import ConnectorSyncBody++ body = ConnectorSyncBody()+ assert body.shared is False+++def test_connector_sync_body_shared_true():+ from api.connectors import ConnectorSyncBody++ body = ConnectorSyncBody(shared=True)+ assert body.shared is True+++def test_connector_sync_body_shared_backwards_compat():+ """Existing clients that omit shared get False."""+ from api.connectors import ConnectorSyncBody++ body = ConnectorSyncBody(selected_files=["file-1"])+ assert body.shared is False+++# ---------------------------------------------------------------------------+# connector_sync guard: shared=True rejected for non-COS connectors+# ---------------------------------------------------------------------------+++async def test_non_cos_connector_rejects_shared_true():+ """The API handler must return 400 when shared=True and connector_type != ibm_cos."""+ from unittest.mock import AsyncMock, MagicMock++ from api.connectors import ConnectorSyncBody, connector_sync++ body = ConnectorSyncBody(shared=True)+ connector_service = MagicMock()+ session_manager = MagicMock()+ user = MagicMock()+ user.jwt_token = "token"+ user.user_id = "user-1"++ # Stub out the connections lookup so the guard fires before any connector work+ connector_service.connection_manager.list_connections = AsyncMock(return_value=[])++ response = await connector_sync(+ connector_type="google_drive",+ body=body,+ connector_service=connector_service,+ session_manager=session_manager,+ user=user,+ )+ assert response.status_code == 400+ import json++ detail = json.loads(response.body)+ assert "ibm_cos" in detail["error"]+++async def test_ibm_cos_shared_true_does_not_hit_guard():+ """shared=True with ibm_cos should NOT be rejected by the guard."""+ from unittest.mock import AsyncMock, MagicMock++ from api.connectors import ConnectorSyncBody, connector_sync++ body = ConnectorSyncBody(shared=True)+ connector_service = MagicMock()+ session_manager = MagicMock()+ user = MagicMock()+ user.jwt_token = "token"+ user.user_id = "user-1"++ # Return empty active connections so we get 404 (not 400) — guard didn't fire+ connector_service.connection_manager.list_connections = AsyncMock(return_value=[])++ response = await connector_sync(+ connector_type="ibm_cos",+ body=body,+ connector_service=connector_service,+ session_manager=session_manager,+ user=user,+ )+ # 404 = "no active connections" error, not 400 = guard rejection+ assert response.status_code == 404