OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: cos share all (backport of #2050) (#2094)
Edwin Jose31fdf1d97259
diff --git a/.env.example b/.env.exampleindex 6716c271c..28aac7b5a 100644--- a/.env.example+++ b/.env.example@@ -173,6 +173,26 @@ MICROSOFT_GRAPH_OAUTH_CLIENT_SECRET=AWS_ACCESS_KEY_ID=AWS_SECRET_ACCESS_KEY=+# --- IBM Cloud Object Storage connector (Enterprise/SaaS) ------------------+# Enterprise/SaaS-only bucket connector, gated by IBM_AUTH_ENABLED (like the+# AWS S3 / Azure Blob connectors). Credentials are normally entered per-connection+# in the UI; these env vars are OPTIONAL fallbacks / defaults. Two auth modes are+# supported: IAM (api_key + service_instance_id, default) or HMAC (access key ++# secret, S3-compatible -- works against MinIO for local testing).+IBM_COS_ENDPOINT=+IBM_COS_API_KEY=+IBM_COS_SERVICE_INSTANCE_ID=+IBM_COS_HMAC_ACCESS_KEY_ID=+IBM_COS_HMAC_SECRET_ACCESS_KEY=+#+# Local dev / MinIO testing: set OPENRAG_DEV_IBM_COS=true to enable the+# connector without IBM_AUTH_ENABLED. NEVER use in production.+# OPENRAG_DEV_IBM_COS=false+# Frontend counterpart: also set NEXT_PUBLIC_DEV_IBM_COS=true so the Connectors+# tab and admin Connectors Permission page show IBM COS locally (requires a+# frontend restart to take effect -- Next.js reads NEXT_PUBLIC_* at startup).+# NEXT_PUBLIC_DEV_IBM_COS=false+# --- Azure Blob Storage connector (Enterprise/SaaS) ------------------------# Enterprise/SaaS-only bucket connector, gated by IBM_AUTH_ENABLED (like the# AWS S3 / IBM COS connectors). Credentials are normally entered per-connectiondiff --git a/enhancements/connectors/ibm_cos/connector.py b/enhancements/connectors/ibm_cos/connector.pyindex a6de8411a..dd50eb683 100644--- a/enhancements/connectors/ibm_cos/connector.py+++ b/enhancements/connectors/ibm_cos/connector.py@@ -1,4 +1,8 @@-"""IBM Cloud Object Storage connector for OpenRAG."""+"""IBM Cloud Object Storage connector for OpenRAG.++Enterprise/SaaS-only ``bucket``-kind connector, gated by ``IBM_AUTH_ENABLED``+(or ``OPENRAG_DEV_IBM_COS=true`` for local dev/MinIO testing).+"""import mimetypesimport os@@ -6,7 +10,7 @@ from datetime import UTC, datetimefrom posixpath import basenamefrom typing import Any-from config.settings import IBM_AUTH_ENABLED+from config.settings import IBM_AUTH_ENABLED, is_dev_ibm_cos_enabledfrom connectors.base import BaseConnector, ConnectorDocument, DocumentACLfrom utils.logging_config import get_logger@@ -64,7 +68,10 @@ class IBMCOSConnector(BaseConnector):@classmethoddef is_available(cls, manager, user_id=None) -> bool:- return IBM_AUTH_ENABLED+ # Enterprise/SaaS gate is IBM_AUTH_ENABLED, like the other bucket+ # connectors (aws_s3, azure_blob). OPENRAG_DEV_IBM_COS=true bypasses it+ # for local dev (e.g. against MinIO in HMAC mode; never in production).+ return IBM_AUTH_ENABLED or is_dev_ibm_cos_enabled()@classmethoddef register_routes(cls, app) -> None:diff --git a/frontend/lib/brand.ts b/frontend/lib/brand.tsindex 819a05daf..2767086fc 100644--- a/frontend/lib/brand.ts+++ b/frontend/lib/brand.ts@@ -4,6 +4,14 @@ export type Brand = "oss" | "ibm";export const IBM_THEME_DEV = process.env.NEXT_PUBLIC_IBM_THEME_DEV === "true";+/**+ * Local dev: show the IBM COS connector in the Connectors tab / permission+ * admin without IBM_AUTH_ENABLED. Mirrors the backend `OPENRAG_DEV_IBM_COS`+ * bypass (see `enhancements/connectors/ibm_cos/connector.py`). Never enable+ * in production.+ */+export const DEV_IBM_COS = process.env.NEXT_PUBLIC_DEV_IBM_COS === "true";+/** Default when no localStorage — must match `BrandProvider` initial state. */export const DEFAULT_BRAND: Brand = IBM_THEME_DEV ? "ibm" : "oss";@@ -82,7 +90,8 @@ export function isConnectorTypeVisible(isIbmAuthMode,}: { isCloudBrand: boolean; isIbmAuthMode: boolean },): boolean {- if (type === "ibm_cos" || type === "aws_s3") return isIbmAuthMode;+ if (type === "ibm_cos") return isIbmAuthMode || DEV_IBM_COS;+ if (type === "aws_s3") return isIbmAuthMode;if (cloudBrand && type === "onedrive") return false;return true;}diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 949fc06ed..4f04eb405 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -1156,6 +1156,7 @@ async def connector_sync(new_ids,jwt_token=jwt_token,ingest_settings=body.settings,+ shared=body.shared,))if changed_ids:@@ -1167,6 +1168,7 @@ async def connector_sync(jwt_token=jwt_token,ingest_settings=body.settings,replace_duplicates=True,+ shared=body.shared,))else:diff --git a/src/config/settings.py b/src/config/settings.pyindex 151e5112f..b6c1fa0da 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -266,6 +266,17 @@ def is_dev_azure_blob_enabled() -> bool:return raw in ("true", "1", "yes", "on")+def is_dev_ibm_cos_enabled() -> bool:+ """Local dev: enable the IBM COS connector without IBM_AUTH_ENABLED.++ Allows testing the IBM COS connector (e.g. against MinIO in HMAC mode) in a+ local environment where IBM auth is not configured. Never enable in+ production. Requires ``OPENRAG_DEV_IBM_COS=true``.+ """+ raw = os.getenv("OPENRAG_DEV_IBM_COS", "false").strip().lower()+ return raw in ("true", "1", "yes", "on")++def is_azure_blob_enabled() -> bool:"""Feature kill switch for the Azure Blob connector (default: enabled).diff --git a/src/models/processors.py b/src/models/processors.pyindex 76f6b7c06..ef1d7dbe1 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -21,7 +21,7 @@ from utils.file_utils import ()from utils.hash_utils import hash_idfrom utils.logging_config import get_logger-from utils.opensearch_queries import build_filename_search_body+from utils.opensearch_queries import build_filename_search_body, build_replace_filename_queryfrom .tasks import FileTask, TaskStatus, UploadTask@@ -836,6 +836,58 @@ class ConnectorFileProcessor(TaskProcessor):self.connector_type = connector_typeself.shared = shared+ async def _reconcile_shared_owner(self, filename: str) -> None:+ """Update owner fields on already-indexed chunks for `filename` to match+ the connector's current `shared` setting.++ Called on the duplicate/unchanged skip paths below, where a file's+ content and name haven't changed since a prior sync but the connector's+ "Make documents available to all users" setting may have been toggled+ since then. Without this, those chunks would keep whatever owner they+ got on their original ingest forever, since a byte-identical re-sync+ never reaches resolve_shared_owner_fields(). Scoped to chunks owned by+ this user or already ownerless (matching the same boundary+ delete_document_by_filename uses), so it can't touch another user's+ private document that happens to share this filename.+ """+ write_client = clients.opensearch+ if write_client is None:+ return+ owner, owner_name, owner_email = resolve_shared_owner_fields(+ self.user_id, self.owner_name, self.owner_email, self.shared+ )+ for candidate in get_filename_aliases(filename):+ try:+ await write_client.update_by_query(+ index=get_index_name(),+ body={+ "query": build_replace_filename_query(candidate, self.user_id),+ "script": {+ "source": """+ if (params.shared) {+ ctx._source.remove('owner');+ } else {+ ctx._source.owner = params.owner;+ }+ ctx._source.owner_name = params.owner_name;+ ctx._source.owner_email = params.owner_email;+ """,+ "params": {+ "shared": self.shared,+ "owner": owner,+ "owner_name": owner_name,+ "owner_email": owner_email,+ },+ },+ },+ )+ except Exception as e:+ logger.warning(+ "Failed to reconcile owner fields for skipped duplicate",+ filename=candidate,+ error=str(e),+ )+async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:"""Process a connector file using unified methods"""file_task.status = TaskStatus.RUNNING@@ -984,6 +1036,7 @@ class ConnectorFileProcessor(TaskProcessor):if await self.check_filename_exists(file_task.filename, opensearch_client):if not self.replace_duplicates:+ await self._reconcile_shared_owner(file_task.filename)file_task.status = TaskStatus.SKIPPEDfile_task.error = Nonefile_task.result = {@@ -1014,6 +1067,7 @@ class ConnectorFileProcessor(TaskProcessor):file_hash = hash_id(tmp_path)if not renamed and await self.check_document_exists(file_hash, opensearch_client):+ await self._reconcile_shared_owner(file_task.filename)file_task.status = TaskStatus.COMPLETEDfile_task.result = {"status": "unchanged", "id": file_hash}file_task.updated_at = time.time()diff --git a/tests/unit/connectors/test_ibm_cos_connector.py b/tests/unit/connectors/test_ibm_cos_connector.pynew file mode 100644index 000000000..bb7e2fac7--- /dev/null+++ b/tests/unit/connectors/test_ibm_cos_connector.py@@ -0,0 +1,41 @@+"""Unit tests for the IBM COS connector's `is_available` gating.++Covers IBM_AUTH_ENABLED / OPENRAG_DEV_IBM_COS, mirroring the equivalent+Azure Blob coverage (tests/unit/connectors/test_azure_blob_connector.py).+"""++import sys+from pathlib import Path+from unittest.mock import MagicMock++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++from enhancements.connectors.ibm_cos import connector as ibm_cos_connector # noqa: E402+from enhancements.connectors.ibm_cos.connector import IBMCOSConnector # noqa: E402+++def test_is_available_gated_on_ibm_auth_enabled(monkeypatch):+ monkeypatch.setattr(ibm_cos_connector, "IBM_AUTH_ENABLED", True)+ monkeypatch.setattr(ibm_cos_connector, "is_dev_ibm_cos_enabled", lambda: False)+ assert IBMCOSConnector.is_available(MagicMock()) is True+ monkeypatch.setattr(ibm_cos_connector, "IBM_AUTH_ENABLED", False)+ assert IBMCOSConnector.is_available(MagicMock()) is False+++def test_is_available_dev_flag_bypasses_ibm_auth(monkeypatch):+ monkeypatch.setattr(ibm_cos_connector, "IBM_AUTH_ENABLED", False)+ monkeypatch.setattr(ibm_cos_connector, "is_dev_ibm_cos_enabled", lambda: True)+ assert IBMCOSConnector.is_available(MagicMock()) is True+++def test_is_dev_ibm_cos_enabled_defaults_false_when_unset(monkeypatch):+ """Regression: must default to disabled, or IBM COS becomes available in+ every deployment (including production) whenever OPENRAG_DEV_IBM_COS is+ simply unset, defeating the point of the dev-only bypass."""+ from config.settings import is_dev_ibm_cos_enabled++ monkeypatch.delenv("OPENRAG_DEV_IBM_COS", raising=False)+ assert is_dev_ibm_cos_enabled() is Falsediff --git a/tests/unit/test_connector_sync_bucket_filter_shared.py b/tests/unit/test_connector_sync_bucket_filter_shared.pynew file mode 100644index 000000000..5307edbed--- /dev/null+++ b/tests/unit/test_connector_sync_bucket_filter_shared.py@@ -0,0 +1,88 @@+"""Regression test: connector_sync's bucket_filter branch must forward+`shared` to both sync_specific_files batches (new + changed files).++This is the code path the COS connector UI's "Ingest N Buckets" button+always hits (it sends bucket_filter, never selected_files/sync_all), so a+dropped `shared` here means the "Make documents available to all users"+toggle silently has no effect on real syncs.+"""++from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from api.connectors import ConnectorSyncBody, connector_sync+++def _make_connection():+ connection = MagicMock()+ connection.connection_id = "conn-1"+ connection.is_active = True+ return connection+++def _make_connector():+ connector = MagicMock()+ connector.authenticate = AsyncMock(return_value=True)+ connector.bucket_names = []+ connector.list_files = AsyncMock(+ return_value={+ "files": [+ {"id": "new-file-1", "modified_time": None},+ {"id": "changed-file-1", "modified_time": None},+ ],+ "next_page_token": None,+ }+ )+ return connector+++async def test_bucket_filter_sync_forwards_shared_to_new_and_changed_batches():+ body = ConnectorSyncBody(bucket_filter=["bucket-a"], shared=True)++ connector_service = MagicMock()+ connection = _make_connection()+ connector_service.connection_manager.list_connections = AsyncMock(return_value=[connection])+ connector = _make_connector()+ connector_service.get_connector = AsyncMock(return_value=connector)+ connector_service.sync_specific_files = AsyncMock(side_effect=["task-new", "task-changed"])++ session_manager = MagicMock()+ user = MagicMock()+ user.jwt_token = "token"+ user.user_id = "user-1"++ with (+ patch(+ "api.connectors.get_synced_file_ids_for_connector",+ new=AsyncMock(return_value=(["changed-file-1"], [], "connector_file_id")),+ ),+ patch(+ "api.connectors.get_synced_id_to_modified_time_map",+ new=AsyncMock(return_value={"changed-file-1": 0.0}),+ ),+ patch(+ "api.connectors.classify_remote_file_change",+ side_effect=lambda fid, *_a, **_k: "new" if fid == "new-file-1" else "changed",+ ),+ ):+ response = await connector_sync(+ connector_type="ibm_cos",+ body=body,+ request=MagicMock(),+ connector_service=connector_service,+ session_manager=session_manager,+ user=user,+ session=AsyncMock(),+ rbac=MagicMock(),+ )++ assert response.status_code == 201+ assert connector_service.sync_specific_files.await_count == 2++ new_call, changed_call = connector_service.sync_specific_files.await_args_list+ assert new_call.args[2] == ["new-file-1"]+ assert new_call.kwargs["shared"] is True+ assert changed_call.args[2] == ["changed-file-1"]+ assert changed_call.kwargs["shared"] is Truediff --git a/tests/unit/test_reconcile_shared_owner.py b/tests/unit/test_reconcile_shared_owner.pynew file mode 100644index 000000000..4aa6e43ad--- /dev/null+++ b/tests/unit/test_reconcile_shared_owner.py@@ -0,0 +1,90 @@+"""Unit tests for ConnectorFileProcessor._reconcile_shared_owner.++Covers the bug where re-syncing a COS bucket with "Make documents available+to all users" newly toggled left already-indexed (unchanged content /+duplicate filename) chunks with stale, non-shared ownership, since the+resolve_shared_owner_fields() recompute is normally only reached on a fresh+index write.+"""++from unittest.mock import AsyncMock++import pytest++import config.settings as settings_module+from models.processors import ConnectorFileProcessor+++def _make_processor(*, shared: bool, user_id: str = "user-1"):+ return ConnectorFileProcessor(+ connector_service=None,+ connection_id="conn-1",+ files_to_process=[],+ user_id=user_id,+ owner_name="Alice",+ owner_email="[email protected]",+ shared=shared,+ )++[email protected](autouse=True)+def _restore_write_client():+ original = settings_module.clients.opensearch+ yield+ settings_module.clients.opensearch = original+++async def test_reconcile_shared_true_omits_owner_via_script():+ write_client = AsyncMock()+ settings_module.clients.opensearch = write_client+ processor = _make_processor(shared=True)++ await processor._reconcile_shared_owner("report.pdf")++ assert write_client.update_by_query.await_count >= 1+ call = write_client.update_by_query.await_args_list[0]+ body = call.kwargs["body"]+ assert body["query"]["bool"]["filter"][0] == {"term": {"filename": "report.pdf"}}+ params = body["script"]["params"]+ assert params["shared"] is True+ assert params["owner"] is None+ assert params["owner_name"] == "Anonymous User"+ assert params["owner_email"] == "anonymous@localhost"+ assert "ctx._source.remove('owner')" in body["script"]["source"]+++async def test_reconcile_shared_false_sets_owner_via_script():+ write_client = AsyncMock()+ settings_module.clients.opensearch = write_client+ processor = _make_processor(shared=False)++ await processor._reconcile_shared_owner("report.pdf")++ call = write_client.update_by_query.await_args_list[0]+ params = call.kwargs["body"]["script"]["params"]+ assert params["shared"] is False+ assert params["owner"] == "user-1"+ assert params["owner_name"] == "Alice"+ assert params["owner_email"] == "[email protected]"+++async def test_reconcile_shared_owner_noop_without_write_client():+ settings_module.clients.opensearch = None+ processor = _make_processor(shared=True)++ # Must not raise even though there's nothing to write to.+ await processor._reconcile_shared_owner("report.pdf")+++async def test_reconcile_shared_owner_swallows_update_errors():+ write_client = AsyncMock()+ write_client.update_by_query.side_effect = RuntimeError("boom")+ settings_module.clients.opensearch = write_client+ processor = _make_processor(shared=True)++ # A failed reconciliation must not fail the (already-skipped) sync item.+ await processor._reconcile_shared_owner("report.pdf")diff --git a/tests/unit/test_shared_flag.py b/tests/unit/test_shared_flag.pyindex 8fc7f1c80..a6c541de6 100644--- a/tests/unit/test_shared_flag.py+++ b/tests/unit/test_shared_flag.py@@ -198,9 +198,12 @@ async def test_non_cos_connector_rejects_shared_true():response = await connector_sync(connector_type="google_drive",body=body,+ request=MagicMock(),connector_service=connector_service,session_manager=session_manager,user=user,+ session=AsyncMock(),+ rbac=MagicMock(),)assert response.status_code == 400import json@@ -229,9 +232,12 @@ async def test_ibm_cos_shared_true_does_not_hit_guard():response = await connector_sync(connector_type="ibm_cos",body=body,+ request=MagicMock(),connector_service=connector_service,session_manager=session_manager,user=user,+ session=AsyncMock(),+ rbac=MagicMock(),)# 404 = "no active connections" error, not 400 = guard rejectionassert response.status_code == 404