OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
feat: add support for custom field parameter in chunk deletion and indexing (#1694)
Rico Furtadoab8ae08e7d39
diff --git a/src/api/connectors.py b/src/api/connectors.pyindex bd5c7dac7..df8acf690 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -29,22 +29,32 @@ async def get_synced_file_ids_for_connector(user_id: str,session_manager,jwt_token: str = None,-) -> tuple:+) -> tuple[list[str], list[str], str]:"""- Query OpenSearch for unique document_id values where connector_type matches.- Returns tuple of (file_ids, filenames) - use file_ids if available, else filenames as fallback.-- Note: Langflow-ingested files may not have document_id stored. In that case,- filenames are returned for filename-based filtering during sync.+ Query OpenSearch for unique file IDs where connector_type matches.++ Returns a 3-tuple ``(file_ids, filenames, id_field)``:++ - ``file_ids``: connector source IDs to use for orphan detection and sync.+ Comes from the ``connector_file_id`` field when chunks were indexed via+ ``ConnectorFileProcessor`` (non-Langflow path); falls back to ``document_id``+ for Langflow-indexed chunks where ``document_id`` already holds the connector+ source ID.+ - ``filenames``: unique filenames as a fallback when ``file_ids`` is empty.+ - ``id_field``: the OpenSearch field name that ``file_ids`` came from+ (``"connector_file_id"`` or ``"document_id"``). Callers must pass this to+ ``delete_orphan_documents`` so deletions target the correct field."""try:opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)- # Query for both document_id and filename aggregationsquery_body = {"size": 0,"query": {"term": {"connector_type": connector_type}},"aggs": {+ "unique_connector_file_ids": {+ "terms": {"field": "connector_file_id", "size": 10000}+ },"unique_document_ids": {"terms": {"field": "document_id", "size": 10000}},"unique_filenames": {"terms": {"field": "filename", "size": 10000}},},@@ -52,26 +62,38 @@ async def get_synced_file_ids_for_connector(result = await opensearch_client.search(index=get_index_name(), body=query_body)- # Get document_ids (preferred - these are the actual connector file IDs)- doc_id_buckets = (- result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])+ # Prefer connector_file_id — these are set by ConnectorFileProcessor (non-Langflow)+ # and hold the actual connector source IDs (e.g. SharePoint GUIDs), not SHA hashes.+ connector_file_id_buckets = (+ result.get("aggregations", {}).get("unique_connector_file_ids", {}).get("buckets", []))- file_ids = [bucket["key"] for bucket in doc_id_buckets if bucket["key"]]+ connector_file_ids = [b["key"] for b in connector_file_id_buckets if b["key"]]++ if connector_file_ids:+ file_ids = connector_file_ids+ id_field = "connector_file_id"+ else:+ # Langflow path: document_id already holds the connector source ID.+ doc_id_buckets = (+ result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])+ )+ file_ids = [b["key"] for b in doc_id_buckets if b["key"]]+ id_field = "document_id"- # Get filenames as fallbackfilename_buckets = (result.get("aggregations", {}).get("unique_filenames", {}).get("buckets", []))- filenames = [bucket["key"] for bucket in filename_buckets if bucket["key"]]+ filenames = [b["key"] for b in filename_buckets if b["key"]]logger.debug("Found synced files for connector",connector_type=connector_type,file_ids_count=len(file_ids),+ id_field=id_field,filenames_count=len(filenames),)- return file_ids, filenames+ return file_ids, filenames, id_fieldexcept Exception as e:logger.error(@@ -79,7 +101,7 @@ async def get_synced_file_ids_for_connector(connector_type=connector_type,error=str(e),)- return [], []+ return [], [], "document_id"async def get_synced_id_to_filename_map(@@ -229,20 +251,31 @@ async def delete_orphan_documents(user_id: str,session_manager,jwt_token: str | None,+ id_field: str = "document_id",) -> int:- """Delete OpenSearch chunks for the given orphan document IDs. Returns the- number of chunks deleted (0 on failure)."""+ """Delete OpenSearch chunks for the given orphan IDs. Returns the number of+ chunks deleted (0 on failure).++ ``id_field`` must match the OpenSearch field that ``orphan_ids`` came from —+ either ``"connector_file_id"`` (ConnectorFileProcessor / non-Langflow path)+ or ``"document_id"`` (Langflow path, where document_id holds the connector+ source ID). Pass the value returned as the third element of+ ``get_synced_file_ids_for_connector()``.+ """if not orphan_ids:return 0from .documents import delete_chunks_by_document_idstry:opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)- return await delete_chunks_by_document_ids(orphan_ids, opensearch_client, get_index_name())+ return await delete_chunks_by_document_ids(+ orphan_ids, opensearch_client, get_index_name(), field=id_field+ )except Exception as e:logger.error("Orphan delete failed",orphan_count=len(orphan_ids),+ id_field=id_field,error=str(e),)return 0@@ -255,11 +288,16 @@ async def reconcile_orphans_for_connector_type(session_manager,jwt_token: str | None,existing_file_ids: list[str],+ id_field: str = "document_id",) -> list[str]:"""Compute and delete orphans for a connector type. Thin wrapper aroundcompute_orphans_for_connector_type + delete_orphan_documents preserved forcallers that perform sync immediately after reconcile.+ ``id_field`` must match the OpenSearch field that ``existing_file_ids`` came+ from. Pass the value returned as the third element of+ ``get_synced_file_ids_for_connector()``.+Returns the list of orphan file IDs that were deleted (or [])."""orphans = await compute_orphans_for_connector_type(@@ -279,12 +317,14 @@ async def reconcile_orphans_for_connector_type(user_id=user_id,session_manager=session_manager,jwt_token=jwt_token,+ id_field=id_field,)logger.info("Orphan reconcile complete",connector_type=connector_type,orphan_count=len(orphan_ids),deleted_chunks=deleted,+ id_field=id_field,)return orphan_ids@@ -472,7 +512,11 @@ async def connector_sync(else:# No files specified - sync only files already in OpenSearch for this connector# This ensures deleted files stay deleted- existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(+ (+ existing_file_ids,+ existing_filenames,+ id_field,+ ) = await get_synced_file_ids_for_connector(connector_type=connector_type,user_id=user.user_id,session_manager=session_manager,@@ -488,13 +532,14 @@ async def connector_sync(status_code=200,)- # If we have document_ids (connector file IDs), use sync_specific_files+ # If we have connector file IDs, use sync_specific_files# Otherwise, use filename filtering with sync_connector_filesif existing_file_ids:logger.info(- "Syncing specific files by document_id",+ "Syncing specific files by connector file ID",connector_type=connector_type,file_count=len(existing_file_ids),+ id_field=id_field,)# Reconcile orphans (files deleted at the source) before re-syncing.# Strict gating: skip when sync is capped — we'd see a partial remote@@ -507,6 +552,7 @@ async def connector_sync(session_manager=session_manager,jwt_token=jwt_token,existing_file_ids=existing_file_ids,+ id_field=id_field,)task_id = await connector_service.sync_specific_files(working_connection.connection_id,@@ -912,7 +958,11 @@ async def sync_all_connectors(for connector_type in cloud_connector_types:try:# First, get existing file IDs/filenames from OpenSearch for this connector type- existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(+ (+ existing_file_ids,+ existing_filenames,+ id_field,+ ) = await get_synced_file_ids_for_connector(connector_type=connector_type,user_id=user.user_id,session_manager=session_manager,@@ -963,12 +1013,13 @@ async def sync_all_connectors()continue- # Sync using document_ids if available, else use filename filter+ # Sync using connector file IDs if available, else use filename filterif existing_file_ids:logger.info(- "Syncing specific files by document_id",+ "Syncing specific files by connector file ID",connector_type=connector_type,file_count=len(existing_file_ids),+ id_field=id_field,)# Reconcile orphans (files deleted at the source) before re-syncing.# sync_all_connectors has no caps or filters, so gating reduces@@ -980,6 +1031,7 @@ async def sync_all_connectors(session_manager=session_manager,jwt_token=jwt_token,existing_file_ids=existing_file_ids,+ id_field=id_field,)task_id = await connector_service.sync_specific_files(working_connection.connection_id,@@ -1076,7 +1128,7 @@ async def _preview_orphans_for_connector_type(Returns (orphans, synced_count). `orphans` is None when strict gating aborts(so the caller can surface a "couldn't determine" state); [] when no orphans."""- existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(+ existing_file_ids, existing_filenames, _ = await get_synced_file_ids_for_connector(connector_type=connector_type,user_id=user_id,session_manager=session_manager,@@ -1363,7 +1415,7 @@ async def browse_connection_files(remote_files = [f for f in remote_files if search_lower in f.get("name", "").lower()]# Get already-ingested file IDs from OpenSearch- ingested_ids, ingested_filenames = await get_synced_file_ids_for_connector(+ ingested_ids, ingested_filenames, _ = await get_synced_file_ids_for_connector(connector_type=connector_type,user_id=user.user_id,session_manager=session_manager,diff --git a/src/api/documents.py b/src/api/documents.pyindex 3a7254930..bd92234e2 100644--- a/src/api/documents.py+++ b/src/api/documents.py@@ -126,13 +126,19 @@ async def delete_chunks_by_document_ids(document_ids: list[str],opensearch_client,index_name: str,+ field: str = "document_id",) -> int:- """Bulk delete OpenSearch chunks by document_id. Returns deleted count.+ """Bulk delete OpenSearch chunks by a keyword field. Returns deleted count.DLS-safe: enumerate the visible chunk _ids via search, then issue a singledelete per primary id. `delete_by_query` is silently no-opped under DLS(returns deleted:N but leaves docs in place — confirmed in the SharePointrename repro debug log).++ `field` selects which indexed keyword to match against (default: ``document_id``).+ Pass ``field="connector_file_id"`` to clean up chunks for a deleted connector+ source file when the connector file ID differs from the content hash stored in+ ``document_id``."""if not document_ids:return 0@@ -141,7 +147,7 @@ async def delete_chunks_by_document_ids(chunk_ids = await collect_visible_document_ids(opensearch_client,index=index_name,- query={"terms": {"document_id": document_ids}},+ query={"terms": {field: document_ids}},)return await delete_document_ids(opensearch_client,diff --git a/src/config/settings.py b/src/config/settings.pyindex e8f3aabee..234ef4127 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -309,6 +309,7 @@ INDEX_BODY = {"embedding_model": {"type": "keyword"},"source_url": {"type": "keyword"},"connector_type": {"type": "keyword"},+ "connector_file_id": {"type": "keyword"},"owner": {"type": "keyword"},"allowed_users": {"type": "keyword"},"allowed_groups": {"type": "keyword"},diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.pyindex a656730a1..44a5a0a02 100644--- a/src/connectors/google_drive/connector.py+++ b/src/connectors/google_drive/connector.py@@ -316,7 +316,7 @@ class GoogleDriveConnector(BaseConnector):fileId=file_id,fields=("id, name, mimeType, modifiedTime, createdTime, size, "- "webViewLink, parents, shortcutDetails, driveId"+ "webViewLink, parents, shortcutDetails, driveId, trashed"),**self._drives_get_flags,)@@ -368,7 +368,7 @@ class GoogleDriveConnector(BaseConnector):)for fid in self.cfg.file_ids:meta = self._get_file_meta_by_id(fid)- if not meta:+ if not meta or meta.get("trashed"):logger.debug("[GoogleDrive] _iter_selected_items: no metadata for file_id=%s", fid)@@ -565,11 +565,6 @@ class GoogleDriveConnector(BaseConnector):logger.debug("[GoogleDrive] authenticate: building Drive service")self.service = self.oauth.get_service()- logger.debug("[GoogleDrive] authenticate: running sanity check (files.get root)")- req = self.service.files().get(fileId="root", fields="id")- await asyncio.to_thread(req.execute)- logger.debug("[GoogleDrive] authenticate: sanity check passed")-self._authenticated = Truereturn Truediff --git a/src/models/processors.py b/src/models/processors.pyindex 84f4d647a..384f0ec05 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -195,6 +195,7 @@ class TaskProcessor:chunk_overlap: int = None,is_sample_data: bool = False,acl: "DocumentACL" = None,+ connector_file_id: str | None = None,):"""Standard processing pipeline for non-Langflow processors:@@ -356,6 +357,8 @@ class TaskProcessor:"connector_type": connector_type,"indexed_time": datetime.datetime.now().isoformat(),}+ if connector_file_id:+ chunk_doc["connector_file_id"] = connector_file_id# Set owner and ACL fieldsif acl:@@ -537,9 +540,10 @@ class ConnectorFileProcessor(TaskProcessor):except (FileNotFoundError, ValueError) as e:msg = str(e).lower()if "not found" in msg or "404" in msg:- # File gone at source — remove indexed chunks by document_id- # (= connector file_id) so it stops appearing in search/chat.- # Filename rename (e.g. .txt → .md) is irrelevant here.+ # File gone at source — remove indexed chunks by connector_file_id+ # so they stop appearing in search/chat. Chunks are indexed with+ # document_id=file_hash (SHA of content), not file_id, so we must+ # query the dedicated connector_file_id field set at index time.deleted_chunks = 0try:from api.documents import delete_chunks_by_document_ids@@ -550,7 +554,10 @@ class ConnectorFileProcessor(TaskProcessor):))deleted_chunks = await delete_chunks_by_document_ids(- [file_id], opensearch_client, get_index_name()+ [file_id],+ opensearch_client,+ get_index_name(),+ field="connector_file_id",)except Exception as cleanup_err:logger.error(@@ -638,6 +645,7 @@ class ConnectorFileProcessor(TaskProcessor):file_size=len(document.content),connector_type=connection.connector_type,acl=document.acl,+ connector_file_id=document.id,**standard_kwargs,)diff --git a/tests/unit/api/test_delete_chunks_by_document_ids.py b/tests/unit/api/test_delete_chunks_by_document_ids.pyindex 5ae43fc49..210e79612 100644--- a/tests/unit/api/test_delete_chunks_by_document_ids.py+++ b/tests/unit/api/test_delete_chunks_by_document_ids.py@@ -85,3 +85,30 @@ async def test_returns_zero_when_no_visible_chunks_match():assert deleted == 0opensearch_client.delete.assert_not_awaited()+++async def test_custom_field_parameter_is_used_in_query():+ """Passing field='connector_file_id' must produce a terms query against that+ field instead of the default 'document_id'. This is required for the 404+ cleanup path in ConnectorFileProcessor where chunks are indexed with+ document_id=file_hash (SHA) but deleted by connector source file ID."""+ from api.documents import delete_chunks_by_document_ids++ opensearch_client = AsyncMock()+ opensearch_client.search.return_value = {+ "_scroll_id": None,+ "hits": {"hits": [{"_id": "chunk-x"}]},+ }+ opensearch_client.delete.return_value = {"result": "deleted"}++ deleted = await delete_chunks_by_document_ids(+ ["src-file-1"],+ opensearch_client,+ "test-index",+ field="connector_file_id",+ )++ assert deleted == 1+ search_call = opensearch_client.search.await_args+ assert search_call.kwargs["body"]["query"] == {"terms": {"connector_file_id": ["src-file-1"]}}diff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.pyindex bd9a1a187..c7f2a186f 100644--- a/tests/unit/api/test_reconcile_orphans_for_connector_type.py+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py@@ -8,6 +8,10 @@ function must:exception aborts the pass with 0 deletes (false-negative > false-positive).- Preserve files that exist in any active connection of the type(multi-connection isolation).+- Delete orphan chunks using the DLS-safe enumerate-then-delete pattern+ (NOT delete_by_query, which is silently no-opped under DLS).+- Query the correct OpenSearch field: connector_file_id for non-Langflow+ chunks, document_id for Langflow chunks."""import sys@@ -24,14 +28,10 @@ if str(SRC) not in sys.path:def _make_connection(connection_id: str, is_active: bool = True):- """Lightweight stand-in for ConnectionConfig — only the attributes- reconcile_orphans_for_connector_type reads."""return SimpleNamespace(connection_id=connection_id, is_active=is_active)def _make_connector(remote_file_ids, *, authenticated=True, raise_on_list=False):- """Build an AsyncMock connector that reports the given file IDs from- list_files() (single page, no pagination)."""connector = MagicMock()connector.is_authenticated = authenticatedif raise_on_list:@@ -44,11 +44,6 @@ def _make_connector(remote_file_ids, *, authenticated=True, raise_on_list=False)def _make_service(connections, connector_lookup):- """Build a connector_service stub.-- `connections` is the list returned by list_connections.- `connector_lookup` maps connection_id -> connector mock (or None).- """service = MagicMock()service.connection_manager = MagicMock()service.connection_manager.list_connections = AsyncMock(return_value=connections)@@ -60,14 +55,33 @@ def _make_service(connections, connector_lookup):return service+def _make_opensearch_client(chunk_ids: list[str] | None = None):+ """Build an OpenSearch mock wired for the enumerate-then-delete pattern.++ ``chunk_ids`` controls which chunk _ids the search returns (simulating+ what collect_visible_document_ids finds). Pass ``None`` or ``[]`` for+ "no chunks found" (no-orphan scenarios).+ """+ client = AsyncMock()+ hits = [{"_id": cid} for cid in (chunk_ids or [])]+ client.search = AsyncMock(return_value={"_scroll_id": None, "hits": {"hits": hits}})+ client.delete = AsyncMock(return_value={"result": "deleted"})+ return client++def _make_session_manager(opensearch_client):sm = MagicMock()sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)return sm+# ---------------------------------------------------------------------------+# Gating / no-op scenarios+# ---------------------------------------------------------------------------++@pytest.mark.asyncio-async def test_empty_existing_file_ids_returns_zero_without_calls():+async def test_empty_existing_file_ids_returns_empty_without_calls():from api.connectors import reconcile_orphans_for_connector_typeservice = MagicMock()@@ -75,7 +89,7 @@ async def test_empty_existing_file_ids_returns_zero_without_calls():service.connection_manager.list_connections = AsyncMock()sm = _make_session_manager(AsyncMock())- deleted = await reconcile_orphans_for_connector_type(+ result = await reconcile_orphans_for_connector_type(connector_type="sharepoint",user_id="alice",connector_service=service,@@ -84,21 +98,20 @@ async def test_empty_existing_file_ids_returns_zero_without_calls():existing_file_ids=[],)- assert deleted == 0+ assert result == []service.connection_manager.list_connections.assert_not_awaited()@pytest.mark.asyncioasync def test_no_active_connections_skips_reconcile():- """If every connection is inactive, we have no remote view — skip."""from api.connectors import reconcile_orphans_for_connector_typeinactive = [_make_connection("c1", is_active=False)]service = _make_service(inactive, connector_lookup={})- opensearch_client = AsyncMock()+ opensearch_client = _make_opensearch_client()sm = _make_session_manager(opensearch_client)- deleted = await reconcile_orphans_for_connector_type(+ result = await reconcile_orphans_for_connector_type(connector_type="sharepoint",user_id="alice",connector_service=service,@@ -107,24 +120,22 @@ async def test_no_active_connections_skips_reconcile():existing_file_ids=["a", "b"],)- assert deleted == 0- opensearch_client.delete_by_query.assert_not_awaited()+ assert result == []+ opensearch_client.delete.assert_not_awaited()@pytest.mark.asyncio-async def test_unauthenticated_connection_aborts_pass(monkeypatch):- """STRICT GATING: even one unauthenticated connector aborts the pass.- Otherwise we'd wrongly mark a file as an orphan because we couldn't- list the connection that holds it."""+async def test_unauthenticated_connection_aborts_pass():+ """STRICT GATING: even one unauthenticated connector aborts the pass."""from api.connectors import reconcile_orphans_for_connector_typeconn = _make_connection("c1")connector = _make_connector(remote_file_ids=[], authenticated=False)service = _make_service([conn], connector_lookup={"c1": connector})- opensearch_client = AsyncMock()+ opensearch_client = _make_opensearch_client()sm = _make_session_manager(opensearch_client)- deleted = await reconcile_orphans_for_connector_type(+ result = await reconcile_orphans_for_connector_type(connector_type="sharepoint",user_id="alice",connector_service=service,@@ -133,24 +144,23 @@ async def test_unauthenticated_connection_aborts_pass(monkeypatch):existing_file_ids=["a", "b"],)- assert deleted == 0+ assert result == []connector.list_files.assert_not_awaited()- opensearch_client.delete_by_query.assert_not_awaited()+ opensearch_client.delete.assert_not_awaited()@pytest.mark.asyncioasync def test_listing_exception_aborts_pass():- """STRICT GATING: a transient list_files error aborts the pass.- Better to leave orphans for one cycle than delete legitimate files."""+ """STRICT GATING: a transient list_files error aborts the pass."""from api.connectors import reconcile_orphans_for_connector_typeconn = _make_connection("c1")connector = _make_connector(remote_file_ids=[], raise_on_list=True)service = _make_service([conn], connector_lookup={"c1": connector})- opensearch_client = AsyncMock()+ opensearch_client = _make_opensearch_client()sm = _make_session_manager(opensearch_client)- deleted = await reconcile_orphans_for_connector_type(+ result = await reconcile_orphans_for_connector_type(connector_type="sharepoint",user_id="alice",connector_service=service,@@ -159,23 +169,57 @@ async def test_listing_exception_aborts_pass():existing_file_ids=["a", "b"],)- assert deleted == 0- opensearch_client.delete_by_query.assert_not_awaited()+ assert result == []+ opensearch_client.delete.assert_not_awaited()+++async def test_no_orphans_skips_delete_call():+ """If every indexed ID still exists remotely, no delete must be issued."""+ from api.connectors import reconcile_orphans_for_connector_type++ conn = _make_connection("c1")+ connector = _make_connector(remote_file_ids=["a", "b"])+ service = _make_service([conn], connector_lookup={"c1": connector})+ opensearch_client = _make_opensearch_client()+ sm = _make_session_manager(opensearch_client)++ result = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["a", "b"],+ )++ assert result == []+ opensearch_client.delete.assert_not_awaited()+++# ---------------------------------------------------------------------------+# Happy path (DLS-safe enumerate-then-delete)+# ---------------------------------------------------------------------------@pytest.mark.asyncioasync def test_happy_path_deletes_orphans():- """Indexed has [a, b, c]; remote has [a, c] -> orphan = [b]."""+ """Indexed has [a, b, c]; remote has [a, c] → orphan = [b].++ The implementation must use the DLS-safe pattern: enumerate visible chunk+ _ids via search, then issue one primary-id delete per chunk. delete_by_query+ must NEVER be called (silently no-opped under DLS).+ """from api.connectors import reconcile_orphans_for_connector_typeconn = _make_connection("c1")connector = _make_connector(remote_file_ids=["a", "c"])service = _make_service([conn], connector_lookup={"c1": connector})- opensearch_client = AsyncMock()- opensearch_client.delete_by_query.return_value = {"deleted": 7}+ # One chunk belongs to orphan "b"+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-0"])sm = _make_session_manager(opensearch_client)- deleted = await reconcile_orphans_for_connector_type(+ result = await reconcile_orphans_for_connector_type(connector_type="sharepoint",user_id="alice",connector_service=service,@@ -184,25 +228,42 @@ async def test_happy_path_deletes_orphans():existing_file_ids=["a", "b", "c"],)- assert deleted == 7- opensearch_client.delete_by_query.assert_awaited_once()- body = opensearch_client.delete_by_query.await_args.kwargs["body"]- assert body == {"query": {"terms": {"document_id": ["b"]}}}+ assert result == ["b"]++ # DLS-safe: must NOT call delete_by_query+ opensearch_client.delete_by_query.assert_not_called()++ # Must enumerate via search first…+ opensearch_client.search.assert_awaited_once()+ search_body = opensearch_client.search.await_args.kwargs["body"]+ assert search_body["query"] == {"terms": {"document_id": ["b"]}}++ # …then delete each chunk by primary _id+ opensearch_client.delete.assert_awaited_once()+ delete_kwargs = opensearch_client.delete.await_args.kwargs+ assert delete_kwargs["id"] == "chunk-b-0"+ assert delete_kwargs.get("refresh") is True@pytest.mark.asyncio-async def test_no_orphans_skips_delete_call():- """If every indexed ID still exists remotely, we must not issue an- empty delete_by_query."""+async def test_delete_failure_does_not_raise():+ """If the bulk delete blows up, the helper must swallow it (no exception).+ The function still returns the orphan IDs that were identified — callers+ use the return value to know *what* was orphaned, not whether deletion+ succeeded."""from api.connectors import reconcile_orphans_for_connector_typeconn = _make_connection("c1")- connector = _make_connector(remote_file_ids=["a", "b"])+ connector = _make_connector(remote_file_ids=["a"])service = _make_service([conn], connector_lookup={"c1": connector})+opensearch_client = AsyncMock()+ # Make the search call itself fail so delete_chunks_by_document_ids raises+ opensearch_client.search = AsyncMock(side_effect=RuntimeError("opensearch unavailable"))sm = _make_session_manager(opensearch_client)- deleted = await reconcile_orphans_for_connector_type(+ # Must not raise even though deletion fails+ result = await reconcile_orphans_for_connector_type(connector_type="sharepoint",user_id="alice",connector_service=service,@@ -211,16 +272,18 @@ async def test_no_orphans_skips_delete_call():existing_file_ids=["a", "b"],)- assert deleted == 0- opensearch_client.delete_by_query.assert_not_awaited()+ # "b" was identified as orphaned; deletion failed silently+ assert result == ["b"]+++# ---------------------------------------------------------------------------+# Multi-connection isolation+# ---------------------------------------------------------------------------@pytest.mark.asyncioasync def test_multi_connection_union_preserves_files_present_in_any_connection():- """Multi-connection isolation:- - User has conn-A and conn-B (both SharePoint, different sites).- - Indexed has [a, b]. conn-A has [a]. conn-B has [b].- - Neither file is an orphan — both should be preserved."""+ """conn-A has [a]. conn-B has [b]. Both [a, b] must be preserved."""from api.connectors import reconcile_orphans_for_connector_typeconn_a = _make_connection("conn-a")@@ -231,10 +294,10 @@ async def test_multi_connection_union_preserves_files_present_in_any_connection([conn_a, conn_b],connector_lookup={"conn-a": connector_a, "conn-b": connector_b},)- opensearch_client = AsyncMock()+ opensearch_client = _make_opensearch_client()sm = _make_session_manager(opensearch_client)- deleted = await reconcile_orphans_for_connector_type(+ result = await reconcile_orphans_for_connector_type(connector_type="sharepoint",user_id="alice",connector_service=service,@@ -243,14 +306,13 @@ async def test_multi_connection_union_preserves_files_present_in_any_connection(existing_file_ids=["a", "b"],)- assert deleted == 0- opensearch_client.delete_by_query.assert_not_awaited()+ assert result == []+ opensearch_client.delete.assert_not_awaited()@pytest.mark.asyncioasync def test_multi_connection_one_offline_aborts_even_if_other_succeeds():- """If conn-A lists fine but conn-B is unauthenticated, we MUST NOT- treat files only in conn-B as orphans. Strict gating bails out."""+ """conn-B unauthenticated — must abort even though conn-A succeeded."""from api.connectors import reconcile_orphans_for_connector_typeconn_a = _make_connection("conn-a")@@ -261,27 +323,25 @@ async def test_multi_connection_one_offline_aborts_even_if_other_succeeds():[conn_a, conn_b],connector_lookup={"conn-a": connector_a, "conn-b": connector_b},)- opensearch_client = AsyncMock()+ opensearch_client = _make_opensearch_client()sm = _make_session_manager(opensearch_client)- deleted = await reconcile_orphans_for_connector_type(+ result = await reconcile_orphans_for_connector_type(connector_type="sharepoint",user_id="alice",connector_service=service,session_manager=sm,jwt_token=None,- existing_file_ids=["a", "b"], # b would look like an orphan if we trusted conn-A alone+ existing_file_ids=["a", "b"],)- assert deleted == 0- opensearch_client.delete_by_query.assert_not_awaited()+ assert result == []+ opensearch_client.delete.assert_not_awaited()@pytest.mark.asyncioasync def test_paginated_listing_aggregates_all_pages():- """Remote listings are paginated. The reconcile must walk every page- before computing the orphan set, otherwise files on page 2 look like- orphans."""+ """Remote listings are paginated — must walk every page."""from api.connectors import reconcile_orphans_for_connector_typeconn = _make_connection("c1")@@ -289,15 +349,15 @@ async def test_paginated_listing_aggregates_all_pages():connector.is_authenticated = Truepages = [{"files": [{"id": "a"}], "nextPageToken": "tok-1"},- {"files": [{"id": "b"}, {"id": "c"}]}, # last page, no token+ {"files": [{"id": "b"}, {"id": "c"}]},]connector.list_files = AsyncMock(side_effect=pages)service = _make_service([conn], connector_lookup={"c1": connector})- opensearch_client = AsyncMock()+ opensearch_client = _make_opensearch_client()sm = _make_session_manager(opensearch_client)- deleted = await reconcile_orphans_for_connector_type(+ result = await reconcile_orphans_for_connector_type(connector_type="sharepoint",user_id="alice",connector_service=service,@@ -306,33 +366,79 @@ async def test_paginated_listing_aggregates_all_pages():existing_file_ids=["a", "b", "c"],)- assert deleted == 0+ assert result == []assert connector.list_files.await_count == 2- opensearch_client.delete_by_query.assert_not_awaited()+ opensearch_client.delete.assert_not_awaited()+++# ---------------------------------------------------------------------------+# id_field routing — connector_file_id vs document_id+# ---------------------------------------------------------------------------@pytest.mark.asyncio-async def test_delete_failure_does_not_raise():- """If the bulk delete itself blows up, the helper must swallow it and- return 0. The caller (sync) should still proceed to re-sync surviving- files — leaving orphans is recoverable, raising is not."""+async def test_connector_file_id_field_used_when_specified():+ """When id_field='connector_file_id', the delete query must target that+ field. This is the non-Langflow (DISABLE_INGEST_WITH_LANGFLOW=True) path+ where chunks carry document_id=SHA_hash but connector_file_id=connector_ID.+ """from api.connectors import reconcile_orphans_for_connector_typeconn = _make_connection("c1")- connector = _make_connector(remote_file_ids=["a"])+ # Remote only has "sp-guid-a"; "sp-guid-b" is orphaned+ connector = _make_connector(remote_file_ids=["sp-guid-a"])service = _make_service([conn], connector_lookup={"c1": connector})+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-0", "chunk-b-1"])+ sm = _make_session_manager(opensearch_client)- opensearch_client = AsyncMock()- opensearch_client.delete_by_query.side_effect = RuntimeError("opensearch unavailable")+ result = await reconcile_orphans_for_connector_type(+ connector_type="sharepoint",+ user_id="alice",+ connector_service=service,+ session_manager=sm,+ jwt_token=None,+ existing_file_ids=["sp-guid-a", "sp-guid-b"],+ id_field="connector_file_id",+ )++ assert result == ["sp-guid-b"]++ opensearch_client.delete_by_query.assert_not_called()++ search_body = opensearch_client.search.await_args.kwargs["body"]+ assert search_body["query"] == {"terms": {"connector_file_id": ["sp-guid-b"]}}, (+ f"Expected connector_file_id query, got: {search_body['query']}"+ )++ assert opensearch_client.delete.await_count == 2+++async def test_document_id_field_used_by_default():+ """Default id_field='document_id' preserves the Langflow path behavior+ where document_id already holds the connector source ID.+ """+ from api.connectors import reconcile_orphans_for_connector_type++ conn = _make_connection("c1")+ connector = _make_connector(remote_file_ids=["lf-id-a"])+ service = _make_service([conn], connector_lookup={"c1": connector})+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-lf-b-0"])sm = _make_session_manager(opensearch_client)- deleted = await reconcile_orphans_for_connector_type(+ result = await reconcile_orphans_for_connector_type(connector_type="sharepoint",user_id="alice",connector_service=service,session_manager=sm,jwt_token=None,- existing_file_ids=["a", "b"],+ existing_file_ids=["lf-id-a", "lf-id-b"],+ # id_field defaults to "document_id")- assert deleted == 0+ assert result == ["lf-id-b"]++ search_body = opensearch_client.search.await_args.kwargs["body"]+ assert search_body["query"] == {"terms": {"document_id": ["lf-id-b"]}}, (+ f"Expected document_id query, got: {search_body['query']}"+ )diff --git a/tests/unit/connectors/test_google_drive_orphan_detection.py b/tests/unit/connectors/test_google_drive_orphan_detection.pynew file mode 100644index 000000000..7c98e6144--- /dev/null+++ b/tests/unit/connectors/test_google_drive_orphan_detection.py@@ -0,0 +1,144 @@+"""Google Drive orphan-detection correctness.++Pins two invariants introduced to fix Google Drive sync:++1. Trashed files must be treated as missing in `_iter_selected_items` when+ iterating the `file_ids` path (the path used by `compute_orphans_for_+ connector_type`). Files moved to Google Drive trash still exist by ID and+ `_get_file_meta_by_id` returns their metadata — without an explicit trash+ check the orphan-detection would consider them still present and fail to+ surface them as orphans.++2. `authenticate()` must not make a live API call. The sanity-check+ (`files().get("root")`) was causing false "connection may need+ re-authentication" warnings when the call failed transiently (rate-limit,+ network blip) even though the OAuth token was valid. Credential validity+ is already ensured by `load_credentials()` and `oauth.is_authenticated()`.+"""++import sys+import threading+from pathlib import Path+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++def _make_connector(file_ids: list[str]):+ """Build a minimal GoogleDriveConnector via __new__, bypassing __init__.++ Only the attributes read by `_iter_selected_items` and `authenticate` are+ set. This avoids the need for real OAuth credentials or token files.+ """+ from connectors.google_drive.connector import GoogleDriveConfig, GoogleDriveConnector++ connector = GoogleDriveConnector.__new__(GoogleDriveConnector)+ connector.cfg = GoogleDriveConfig(+ client_id="fake-client-id",+ client_secret="fake-client-secret",+ token_file="/tmp/fake-token.json",+ file_ids=file_ids,+ folder_ids=None,+ include_mime_types=None,+ exclude_mime_types=None,+ )+ connector.service = MagicMock()+ connector._authenticated = False+ connector._lock = threading.Lock()+ connector._shortcut_cache = {}+ return connector+++# ---------------------------------------------------------------------------+# _iter_selected_items — trashed / missing / present+# ---------------------------------------------------------------------------+++def test_trashed_file_not_in_iter_selected_items():+ """A file that exists in Drive trash must be excluded from orphan listing.++ Without this guard, `_get_file_meta_by_id` returns metadata for trashed+ files (they still exist by ID), causing them to appear in `remote_ids`+ and thus NOT be flagged as orphans.+ """+ connector = _make_connector(file_ids=["trashed-file-id"])++ with patch.object(+ connector,+ "_get_file_meta_by_id",+ return_value={"id": "trashed-file-id", "mimeType": "application/pdf", "trashed": True},+ ):+ result = connector._iter_selected_items()++ assert result == [], "trashed file must not appear in iter_selected_items output"+++def test_non_trashed_file_is_in_iter_selected_items():+ """A file that exists and is not trashed must appear in the results."""+ connector = _make_connector(file_ids=["live-file-id"])++ with patch.object(+ connector,+ "_get_file_meta_by_id",+ return_value={"id": "live-file-id", "mimeType": "application/pdf", "trashed": False},+ ):+ result = connector._iter_selected_items()++ assert len(result) == 1+ assert result[0]["id"] == "live-file-id"+++def test_missing_file_id_not_in_iter_selected_items():+ """A file that returns None from `_get_file_meta_by_id` (e.g. permanently+ deleted or permission error) must not appear in the results.++ Regression guard for the pre-existing `if not meta: continue` behavior.+ """+ connector = _make_connector(file_ids=["gone-file-id"])++ with patch.object(connector, "_get_file_meta_by_id", return_value=None):+ result = connector._iter_selected_items()++ assert result == []+++# ---------------------------------------------------------------------------+# authenticate() — no live API call+# ---------------------------------------------------------------------------+++async def test_authenticate_succeeds_without_live_api_call():+ """authenticate() must not make a live API call to verify credentials.++ The sanity-check `files().get("root").execute()` caused false auth+ failures when the call failed transiently. Removing it means successful+ authentication is determined solely by credential validity — which+ `load_credentials()` and `oauth.is_authenticated()` already ensure.+ """+ from connectors.google_drive.connector import GoogleDriveConnector++ connector = GoogleDriveConnector.__new__(GoogleDriveConnector)+ connector._authenticated = False++ fake_creds = MagicMock()+ fake_service = MagicMock()++ connector.oauth = MagicMock()+ connector.oauth.load_credentials = AsyncMock(return_value=fake_creds)+ connector.oauth.is_authenticated = AsyncMock(return_value=True)+ connector.oauth.get_service = MagicMock(return_value=fake_service)++ result = await connector.authenticate()++ assert result is True+ assert connector._authenticated is True+ # The service must be set — connector is usable after authenticate().+ assert connector.service is fake_service+ # No live API call must have been made.+ fake_service.files.assert_not_called()diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.pyindex 9cdfe1685..a866939e5 100644--- a/tests/unit/test_connector_processor_filename_dedupe.py+++ b/tests/unit/test_connector_processor_filename_dedupe.py@@ -131,8 +131,9 @@ async def test_connector_processor_deletes_then_ingests_when_replace_true():@pytest.mark.asyncioasync def test_connector_processor_deletes_chunks_when_source_returns_404():"""When the source connector reports the file is gone (404), the processor- must remove the already-indexed chunks for that document_id. Regression- test for SharePoint sync leaving orphan chunks after a source-side delete.+ must remove the already-indexed chunks queried by connector_file_id — NOT+ document_id. Chunks are indexed with document_id=file_hash (SHA of content)+ which differs from file_id, so querying document_id would find nothing."""processor = _build_connector_processor(replace_duplicates=False)@@ -168,6 +169,16 @@ async def test_connector_processor_deletes_chunks_when_source_returns_404():assert upload_task.successful_files == 1opensearch_client.delete.assert_awaited_once()+ # The search must target connector_file_id, not document_id.+ # document_id holds a SHA content hash that won't match the connector file ID.+ search_call = opensearch_client.search.await_args+ query = search_call.kwargs["body"]["query"]+ assert "terms" in query, f"expected a terms query, got: {query}"+ assert "connector_file_id" in query["terms"], (+ f"cleanup must query connector_file_id, not document_id. Query: {query}"+ )+ assert query["terms"]["connector_file_id"] == ["file-id-1"]+@pytest.mark.asyncioasync def test_connector_processor_proceeds_when_filename_absent():diff --git a/tests/unit/test_processors_clear_stale_chunks.py b/tests/unit/test_processors_clear_stale_chunks.pyindex 5c0246525..6136e8df0 100644--- a/tests/unit/test_processors_clear_stale_chunks.py+++ b/tests/unit/test_processors_clear_stale_chunks.py@@ -217,3 +217,74 @@ async def test_delete_failure_does_not_abort_reindex(monkeypatch):assert result["status"] == "indexed"assert len(index_calls) == 2, "indexing must still happen if the pre-delete fails"+++async def test_connector_file_id_stored_in_chunk_when_provided(monkeypatch):+ """When connector_file_id is passed to process_document_standard, every+ indexed chunk must carry that value so the 404 cleanup path can find and+ delete chunks by their connector source ID."""+ processor, opensearch_client = _make_processor_with_mocks()+ _patch_embedding_pipeline(monkeypatch, chunk_count=2)++ opensearch_client.search = AsyncMock(return_value={"_scroll_id": None, "hits": {"hits": []}})+ opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})+ indexed_bodies: list[dict] = []+ opensearch_client.index = AsyncMock(+ side_effect=lambda **kw: indexed_bodies.append(kw.get("body", {}))+ )++ with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:+ tmp.write(b"hello world")+ tmp_path = tmp.name++ try:+ result = await processor.process_document_standard(+ file_path=tmp_path,+ file_hash="sha-abc",+ owner_user_id="alice",+ original_filename="report.txt",+ connector_type="sharepoint",+ connector_file_id="sharepoint-item-xyz",+ )+ finally:+ Path(tmp_path).unlink(missing_ok=True)++ assert result["status"] == "indexed"+ assert len(indexed_bodies) == 2, "expected one index call per chunk"+ for body in indexed_bodies:+ assert body.get("connector_file_id") == "sharepoint-item-xyz", (+ f"connector_file_id missing or wrong in chunk: {body}"+ )+ assert body.get("document_id") == "sha-abc", "document_id must still be the content hash"+++async def test_connector_file_id_absent_when_not_provided(monkeypatch):+ """When connector_file_id is not passed (local uploads, non-connector paths),+ the field must be absent from indexed chunks — no None/empty pollution."""+ processor, opensearch_client = _make_processor_with_mocks()+ _patch_embedding_pipeline(monkeypatch, chunk_count=1)++ opensearch_client.search = AsyncMock(return_value={"_scroll_id": None, "hits": {"hits": []}})+ opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})+ indexed_bodies: list[dict] = []+ opensearch_client.index = AsyncMock(+ side_effect=lambda **kw: indexed_bodies.append(kw.get("body", {}))+ )++ with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:+ tmp.write(b"hello world")+ tmp_path = tmp.name++ try:+ await processor.process_document_standard(+ file_path=tmp_path,+ file_hash="sha-xyz",+ owner_user_id="alice",+ )+ finally:+ Path(tmp_path).unlink(missing_ok=True)++ assert len(indexed_bodies) == 1+ assert "connector_file_id" not in indexed_bodies[0]