Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

feat: add support for custom field parameter in chunk deletion and indexing (#1694)

Rico Furtadoab8ae08e7d39
diff --git a/src/api/connectors.py b/src/api/connectors.py
index bd5c7dac7..df8acf690 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -29,22 +29,32 @@ async def get_synced_file_ids_for_connector(
user_id: str,
session_manager,
jwt_token: str = None,
-) -> tuple:
+) -> tuple[list[str], list[str], str]:
"""
- Query OpenSearch for unique document_id values where connector_type matches.
- Returns tuple of (file_ids, filenames) - use file_ids if available, else filenames as fallback.
-
- Note: Langflow-ingested files may not have document_id stored. In that case,
- filenames are returned for filename-based filtering during sync.
+ Query OpenSearch for unique file IDs where connector_type matches.
+
+ Returns a 3-tuple ``(file_ids, filenames, id_field)``:
+
+ - ``file_ids``: connector source IDs to use for orphan detection and sync.
+ Comes from the ``connector_file_id`` field when chunks were indexed via
+ ``ConnectorFileProcessor`` (non-Langflow path); falls back to ``document_id``
+ for Langflow-indexed chunks where ``document_id`` already holds the connector
+ source ID.
+ - ``filenames``: unique filenames as a fallback when ``file_ids`` is empty.
+ - ``id_field``: the OpenSearch field name that ``file_ids`` came from
+ (``"connector_file_id"`` or ``"document_id"``). Callers must pass this to
+ ``delete_orphan_documents`` so deletions target the correct field.
"""
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
- # Query for both document_id and filename aggregations
query_body = {
"size": 0,
"query": {"term": {"connector_type": connector_type}},
"aggs": {
+ "unique_connector_file_ids": {
+ "terms": {"field": "connector_file_id", "size": 10000}
+ },
"unique_document_ids": {"terms": {"field": "document_id", "size": 10000}},
"unique_filenames": {"terms": {"field": "filename", "size": 10000}},
},
@@ -52,26 +62,38 @@ async def get_synced_file_ids_for_connector(
result = await opensearch_client.search(index=get_index_name(), body=query_body)
- # Get document_ids (preferred - these are the actual connector file IDs)
- doc_id_buckets = (
- result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])
+ # Prefer connector_file_id — these are set by ConnectorFileProcessor (non-Langflow)
+ # and hold the actual connector source IDs (e.g. SharePoint GUIDs), not SHA hashes.
+ connector_file_id_buckets = (
+ result.get("aggregations", {}).get("unique_connector_file_ids", {}).get("buckets", [])
)
- file_ids = [bucket["key"] for bucket in doc_id_buckets if bucket["key"]]
+ connector_file_ids = [b["key"] for b in connector_file_id_buckets if b["key"]]
+
+ if connector_file_ids:
+ file_ids = connector_file_ids
+ id_field = "connector_file_id"
+ else:
+ # Langflow path: document_id already holds the connector source ID.
+ doc_id_buckets = (
+ result.get("aggregations", {}).get("unique_document_ids", {}).get("buckets", [])
+ )
+ file_ids = [b["key"] for b in doc_id_buckets if b["key"]]
+ id_field = "document_id"
- # Get filenames as fallback
filename_buckets = (
result.get("aggregations", {}).get("unique_filenames", {}).get("buckets", [])
)
- filenames = [bucket["key"] for bucket in filename_buckets if bucket["key"]]
+ filenames = [b["key"] for b in filename_buckets if b["key"]]
logger.debug(
"Found synced files for connector",
connector_type=connector_type,
file_ids_count=len(file_ids),
+ id_field=id_field,
filenames_count=len(filenames),
)
- return file_ids, filenames
+ return file_ids, filenames, id_field
except Exception as e:
logger.error(
@@ -79,7 +101,7 @@ async def get_synced_file_ids_for_connector(
connector_type=connector_type,
error=str(e),
)
- return [], []
+ return [], [], "document_id"
async def get_synced_id_to_filename_map(
@@ -229,20 +251,31 @@ async def delete_orphan_documents(
user_id: str,
session_manager,
jwt_token: str | None,
+ id_field: str = "document_id",
) -> int:
- """Delete OpenSearch chunks for the given orphan document IDs. Returns the
- number of chunks deleted (0 on failure)."""
+ """Delete OpenSearch chunks for the given orphan IDs. Returns the number of
+ chunks deleted (0 on failure).
+
+ ``id_field`` must match the OpenSearch field that ``orphan_ids`` came from —
+ either ``"connector_file_id"`` (ConnectorFileProcessor / non-Langflow path)
+ or ``"document_id"`` (Langflow path, where document_id holds the connector
+ source ID). Pass the value returned as the third element of
+ ``get_synced_file_ids_for_connector()``.
+ """
if not orphan_ids:
return 0
from .documents import delete_chunks_by_document_ids
try:
opensearch_client = session_manager.get_user_opensearch_client(user_id, jwt_token)
- return await delete_chunks_by_document_ids(orphan_ids, opensearch_client, get_index_name())
+ return await delete_chunks_by_document_ids(
+ orphan_ids, opensearch_client, get_index_name(), field=id_field
+ )
except Exception as e:
logger.error(
"Orphan delete failed",
orphan_count=len(orphan_ids),
+ id_field=id_field,
error=str(e),
)
return 0
@@ -255,11 +288,16 @@ async def reconcile_orphans_for_connector_type(
session_manager,
jwt_token: str | None,
existing_file_ids: list[str],
+ id_field: str = "document_id",
) -> list[str]:
"""Compute and delete orphans for a connector type. Thin wrapper around
compute_orphans_for_connector_type + delete_orphan_documents preserved for
callers that perform sync immediately after reconcile.
+ ``id_field`` must match the OpenSearch field that ``existing_file_ids`` came
+ from. Pass the value returned as the third element of
+ ``get_synced_file_ids_for_connector()``.
+
Returns the list of orphan file IDs that were deleted (or []).
"""
orphans = await compute_orphans_for_connector_type(
@@ -279,12 +317,14 @@ async def reconcile_orphans_for_connector_type(
user_id=user_id,
session_manager=session_manager,
jwt_token=jwt_token,
+ id_field=id_field,
)
logger.info(
"Orphan reconcile complete",
connector_type=connector_type,
orphan_count=len(orphan_ids),
deleted_chunks=deleted,
+ id_field=id_field,
)
return orphan_ids
@@ -472,7 +512,11 @@ async def connector_sync(
else:
# No files specified - sync only files already in OpenSearch for this connector
# This ensures deleted files stay deleted
- existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(
+ (
+ existing_file_ids,
+ existing_filenames,
+ id_field,
+ ) = await get_synced_file_ids_for_connector(
connector_type=connector_type,
user_id=user.user_id,
session_manager=session_manager,
@@ -488,13 +532,14 @@ async def connector_sync(
status_code=200,
)
- # If we have document_ids (connector file IDs), use sync_specific_files
+ # If we have connector file IDs, use sync_specific_files
# Otherwise, use filename filtering with sync_connector_files
if existing_file_ids:
logger.info(
- "Syncing specific files by document_id",
+ "Syncing specific files by connector file ID",
connector_type=connector_type,
file_count=len(existing_file_ids),
+ id_field=id_field,
)
# Reconcile orphans (files deleted at the source) before re-syncing.
# Strict gating: skip when sync is capped — we'd see a partial remote
@@ -507,6 +552,7 @@ async def connector_sync(
session_manager=session_manager,
jwt_token=jwt_token,
existing_file_ids=existing_file_ids,
+ id_field=id_field,
)
task_id = await connector_service.sync_specific_files(
working_connection.connection_id,
@@ -912,7 +958,11 @@ async def sync_all_connectors(
for connector_type in cloud_connector_types:
try:
# First, get existing file IDs/filenames from OpenSearch for this connector type
- existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(
+ (
+ existing_file_ids,
+ existing_filenames,
+ id_field,
+ ) = await get_synced_file_ids_for_connector(
connector_type=connector_type,
user_id=user.user_id,
session_manager=session_manager,
@@ -963,12 +1013,13 @@ async def sync_all_connectors(
)
continue
- # Sync using document_ids if available, else use filename filter
+ # Sync using connector file IDs if available, else use filename filter
if existing_file_ids:
logger.info(
- "Syncing specific files by document_id",
+ "Syncing specific files by connector file ID",
connector_type=connector_type,
file_count=len(existing_file_ids),
+ id_field=id_field,
)
# Reconcile orphans (files deleted at the source) before re-syncing.
# sync_all_connectors has no caps or filters, so gating reduces
@@ -980,6 +1031,7 @@ async def sync_all_connectors(
session_manager=session_manager,
jwt_token=jwt_token,
existing_file_ids=existing_file_ids,
+ id_field=id_field,
)
task_id = await connector_service.sync_specific_files(
working_connection.connection_id,
@@ -1076,7 +1128,7 @@ async def _preview_orphans_for_connector_type(
Returns (orphans, synced_count). `orphans` is None when strict gating aborts
(so the caller can surface a "couldn't determine" state); [] when no orphans.
"""
- existing_file_ids, existing_filenames = await get_synced_file_ids_for_connector(
+ existing_file_ids, existing_filenames, _ = await get_synced_file_ids_for_connector(
connector_type=connector_type,
user_id=user_id,
session_manager=session_manager,
@@ -1363,7 +1415,7 @@ async def browse_connection_files(
remote_files = [f for f in remote_files if search_lower in f.get("name", "").lower()]
# Get already-ingested file IDs from OpenSearch
- ingested_ids, ingested_filenames = await get_synced_file_ids_for_connector(
+ ingested_ids, ingested_filenames, _ = await get_synced_file_ids_for_connector(
connector_type=connector_type,
user_id=user.user_id,
session_manager=session_manager,
diff --git a/src/api/documents.py b/src/api/documents.py
index 3a7254930..bd92234e2 100644
--- a/src/api/documents.py
+++ b/src/api/documents.py
@@ -126,13 +126,19 @@ async def delete_chunks_by_document_ids(
document_ids: list[str],
opensearch_client,
index_name: str,
+ field: str = "document_id",
) -> int:
- """Bulk delete OpenSearch chunks by document_id. Returns deleted count.
+ """Bulk delete OpenSearch chunks by a keyword field. Returns deleted count.
DLS-safe: enumerate the visible chunk _ids via search, then issue a single
delete per primary id. `delete_by_query` is silently no-opped under DLS
(returns deleted:N but leaves docs in place — confirmed in the SharePoint
rename repro debug log).
+
+ `field` selects which indexed keyword to match against (default: ``document_id``).
+ Pass ``field="connector_file_id"`` to clean up chunks for a deleted connector
+ source file when the connector file ID differs from the content hash stored in
+ ``document_id``.
"""
if not document_ids:
return 0
@@ -141,7 +147,7 @@ async def delete_chunks_by_document_ids(
chunk_ids = await collect_visible_document_ids(
opensearch_client,
index=index_name,
- query={"terms": {"document_id": document_ids}},
+ query={"terms": {field: document_ids}},
)
return await delete_document_ids(
opensearch_client,
diff --git a/src/config/settings.py b/src/config/settings.py
index e8f3aabee..234ef4127 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -309,6 +309,7 @@ INDEX_BODY = {
"embedding_model": {"type": "keyword"},
"source_url": {"type": "keyword"},
"connector_type": {"type": "keyword"},
+ "connector_file_id": {"type": "keyword"},
"owner": {"type": "keyword"},
"allowed_users": {"type": "keyword"},
"allowed_groups": {"type": "keyword"},
diff --git a/src/connectors/google_drive/connector.py b/src/connectors/google_drive/connector.py
index a656730a1..44a5a0a02 100644
--- a/src/connectors/google_drive/connector.py
+++ b/src/connectors/google_drive/connector.py
@@ -316,7 +316,7 @@ class GoogleDriveConnector(BaseConnector):
fileId=file_id,
fields=(
"id, name, mimeType, modifiedTime, createdTime, size, "
- "webViewLink, parents, shortcutDetails, driveId"
+ "webViewLink, parents, shortcutDetails, driveId, trashed"
),
**self._drives_get_flags,
)
@@ -368,7 +368,7 @@ class GoogleDriveConnector(BaseConnector):
)
for fid in self.cfg.file_ids:
meta = self._get_file_meta_by_id(fid)
- if not meta:
+ if not meta or meta.get("trashed"):
logger.debug(
"[GoogleDrive] _iter_selected_items: no metadata for file_id=%s", fid
)
@@ -565,11 +565,6 @@ class GoogleDriveConnector(BaseConnector):
logger.debug("[GoogleDrive] authenticate: building Drive service")
self.service = self.oauth.get_service()
- logger.debug("[GoogleDrive] authenticate: running sanity check (files.get root)")
- req = self.service.files().get(fileId="root", fields="id")
- await asyncio.to_thread(req.execute)
- logger.debug("[GoogleDrive] authenticate: sanity check passed")
-
self._authenticated = True
return True
diff --git a/src/models/processors.py b/src/models/processors.py
index 84f4d647a..384f0ec05 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -195,6 +195,7 @@ class TaskProcessor:
chunk_overlap: int = None,
is_sample_data: bool = False,
acl: "DocumentACL" = None,
+ connector_file_id: str | None = None,
):
"""
Standard processing pipeline for non-Langflow processors:
@@ -356,6 +357,8 @@ class TaskProcessor:
"connector_type": connector_type,
"indexed_time": datetime.datetime.now().isoformat(),
}
+ if connector_file_id:
+ chunk_doc["connector_file_id"] = connector_file_id
# Set owner and ACL fields
if acl:
@@ -537,9 +540,10 @@ class ConnectorFileProcessor(TaskProcessor):
except (FileNotFoundError, ValueError) as e:
msg = str(e).lower()
if "not found" in msg or "404" in msg:
- # File gone at source — remove indexed chunks by document_id
- # (= connector file_id) so it stops appearing in search/chat.
- # Filename rename (e.g. .txt → .md) is irrelevant here.
+ # File gone at source — remove indexed chunks by connector_file_id
+ # so they stop appearing in search/chat. Chunks are indexed with
+ # document_id=file_hash (SHA of content), not file_id, so we must
+ # query the dedicated connector_file_id field set at index time.
deleted_chunks = 0
try:
from api.documents import delete_chunks_by_document_ids
@@ -550,7 +554,10 @@ class ConnectorFileProcessor(TaskProcessor):
)
)
deleted_chunks = await delete_chunks_by_document_ids(
- [file_id], opensearch_client, get_index_name()
+ [file_id],
+ opensearch_client,
+ get_index_name(),
+ field="connector_file_id",
)
except Exception as cleanup_err:
logger.error(
@@ -638,6 +645,7 @@ class ConnectorFileProcessor(TaskProcessor):
file_size=len(document.content),
connector_type=connection.connector_type,
acl=document.acl,
+ connector_file_id=document.id,
**standard_kwargs,
)
diff --git a/tests/unit/api/test_delete_chunks_by_document_ids.py b/tests/unit/api/test_delete_chunks_by_document_ids.py
index 5ae43fc49..210e79612 100644
--- a/tests/unit/api/test_delete_chunks_by_document_ids.py
+++ b/tests/unit/api/test_delete_chunks_by_document_ids.py
@@ -85,3 +85,30 @@ async def test_returns_zero_when_no_visible_chunks_match():
assert deleted == 0
opensearch_client.delete.assert_not_awaited()
+
+
+async def test_custom_field_parameter_is_used_in_query():
+ """Passing field='connector_file_id' must produce a terms query against that
+ field instead of the default 'document_id'. This is required for the 404
+ cleanup path in ConnectorFileProcessor where chunks are indexed with
+ document_id=file_hash (SHA) but deleted by connector source file ID."""
+ from api.documents import delete_chunks_by_document_ids
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search.return_value = {
+ "_scroll_id": None,
+ "hits": {"hits": [{"_id": "chunk-x"}]},
+ }
+ opensearch_client.delete.return_value = {"result": "deleted"}
+
+ deleted = await delete_chunks_by_document_ids(
+ ["src-file-1"],
+ opensearch_client,
+ "test-index",
+ field="connector_file_id",
+ )
+
+ assert deleted == 1
+ search_call = opensearch_client.search.await_args
+ assert search_call.kwargs["body"]["query"] == {"terms": {"connector_file_id": ["src-file-1"]}}
diff --git a/tests/unit/api/test_reconcile_orphans_for_connector_type.py b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
index bd9a1a187..c7f2a186f 100644
--- a/tests/unit/api/test_reconcile_orphans_for_connector_type.py
+++ b/tests/unit/api/test_reconcile_orphans_for_connector_type.py
@@ -8,6 +8,10 @@ function must:
exception aborts the pass with 0 deletes (false-negative > false-positive).
- Preserve files that exist in any active connection of the type
(multi-connection isolation).
+- Delete orphan chunks using the DLS-safe enumerate-then-delete pattern
+ (NOT delete_by_query, which is silently no-opped under DLS).
+- Query the correct OpenSearch field: connector_file_id for non-Langflow
+ chunks, document_id for Langflow chunks.
"""
import sys
@@ -24,14 +28,10 @@ if str(SRC) not in sys.path:
def _make_connection(connection_id: str, is_active: bool = True):
- """Lightweight stand-in for ConnectionConfig — only the attributes
- reconcile_orphans_for_connector_type reads."""
return SimpleNamespace(connection_id=connection_id, is_active=is_active)
def _make_connector(remote_file_ids, *, authenticated=True, raise_on_list=False):
- """Build an AsyncMock connector that reports the given file IDs from
- list_files() (single page, no pagination)."""
connector = MagicMock()
connector.is_authenticated = authenticated
if raise_on_list:
@@ -44,11 +44,6 @@ def _make_connector(remote_file_ids, *, authenticated=True, raise_on_list=False)
def _make_service(connections, connector_lookup):
- """Build a connector_service stub.
-
- `connections` is the list returned by list_connections.
- `connector_lookup` maps connection_id -> connector mock (or None).
- """
service = MagicMock()
service.connection_manager = MagicMock()
service.connection_manager.list_connections = AsyncMock(return_value=connections)
@@ -60,14 +55,33 @@ def _make_service(connections, connector_lookup):
return service
+def _make_opensearch_client(chunk_ids: list[str] | None = None):
+ """Build an OpenSearch mock wired for the enumerate-then-delete pattern.
+
+ ``chunk_ids`` controls which chunk _ids the search returns (simulating
+ what collect_visible_document_ids finds). Pass ``None`` or ``[]`` for
+ "no chunks found" (no-orphan scenarios).
+ """
+ client = AsyncMock()
+ hits = [{"_id": cid} for cid in (chunk_ids or [])]
+ client.search = AsyncMock(return_value={"_scroll_id": None, "hits": {"hits": hits}})
+ client.delete = AsyncMock(return_value={"result": "deleted"})
+ return client
+
+
def _make_session_manager(opensearch_client):
sm = MagicMock()
sm.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
return sm
+# ---------------------------------------------------------------------------
+# Gating / no-op scenarios
+# ---------------------------------------------------------------------------
+
+
@pytest.mark.asyncio
-async def test_empty_existing_file_ids_returns_zero_without_calls():
+async def test_empty_existing_file_ids_returns_empty_without_calls():
from api.connectors import reconcile_orphans_for_connector_type
service = MagicMock()
@@ -75,7 +89,7 @@ async def test_empty_existing_file_ids_returns_zero_without_calls():
service.connection_manager.list_connections = AsyncMock()
sm = _make_session_manager(AsyncMock())
- deleted = await reconcile_orphans_for_connector_type(
+ result = await reconcile_orphans_for_connector_type(
connector_type="sharepoint",
user_id="alice",
connector_service=service,
@@ -84,21 +98,20 @@ async def test_empty_existing_file_ids_returns_zero_without_calls():
existing_file_ids=[],
)
- assert deleted == 0
+ assert result == []
service.connection_manager.list_connections.assert_not_awaited()
@pytest.mark.asyncio
async def test_no_active_connections_skips_reconcile():
- """If every connection is inactive, we have no remote view — skip."""
from api.connectors import reconcile_orphans_for_connector_type
inactive = [_make_connection("c1", is_active=False)]
service = _make_service(inactive, connector_lookup={})
- opensearch_client = AsyncMock()
+ opensearch_client = _make_opensearch_client()
sm = _make_session_manager(opensearch_client)
- deleted = await reconcile_orphans_for_connector_type(
+ result = await reconcile_orphans_for_connector_type(
connector_type="sharepoint",
user_id="alice",
connector_service=service,
@@ -107,24 +120,22 @@ async def test_no_active_connections_skips_reconcile():
existing_file_ids=["a", "b"],
)
- assert deleted == 0
- opensearch_client.delete_by_query.assert_not_awaited()
+ assert result == []
+ opensearch_client.delete.assert_not_awaited()
@pytest.mark.asyncio
-async def test_unauthenticated_connection_aborts_pass(monkeypatch):
- """STRICT GATING: even one unauthenticated connector aborts the pass.
- Otherwise we'd wrongly mark a file as an orphan because we couldn't
- list the connection that holds it."""
+async def test_unauthenticated_connection_aborts_pass():
+ """STRICT GATING: even one unauthenticated connector aborts the pass."""
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
connector = _make_connector(remote_file_ids=[], authenticated=False)
service = _make_service([conn], connector_lookup={"c1": connector})
- opensearch_client = AsyncMock()
+ opensearch_client = _make_opensearch_client()
sm = _make_session_manager(opensearch_client)
- deleted = await reconcile_orphans_for_connector_type(
+ result = await reconcile_orphans_for_connector_type(
connector_type="sharepoint",
user_id="alice",
connector_service=service,
@@ -133,24 +144,23 @@ async def test_unauthenticated_connection_aborts_pass(monkeypatch):
existing_file_ids=["a", "b"],
)
- assert deleted == 0
+ assert result == []
connector.list_files.assert_not_awaited()
- opensearch_client.delete_by_query.assert_not_awaited()
+ opensearch_client.delete.assert_not_awaited()
@pytest.mark.asyncio
async def test_listing_exception_aborts_pass():
- """STRICT GATING: a transient list_files error aborts the pass.
- Better to leave orphans for one cycle than delete legitimate files."""
+ """STRICT GATING: a transient list_files error aborts the pass."""
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
connector = _make_connector(remote_file_ids=[], raise_on_list=True)
service = _make_service([conn], connector_lookup={"c1": connector})
- opensearch_client = AsyncMock()
+ opensearch_client = _make_opensearch_client()
sm = _make_session_manager(opensearch_client)
- deleted = await reconcile_orphans_for_connector_type(
+ result = await reconcile_orphans_for_connector_type(
connector_type="sharepoint",
user_id="alice",
connector_service=service,
@@ -159,23 +169,57 @@ async def test_listing_exception_aborts_pass():
existing_file_ids=["a", "b"],
)
- assert deleted == 0
- opensearch_client.delete_by_query.assert_not_awaited()
+ assert result == []
+ opensearch_client.delete.assert_not_awaited()
+
+
+async def test_no_orphans_skips_delete_call():
+ """If every indexed ID still exists remotely, no delete must be issued."""
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = _make_connector(remote_file_ids=["a", "b"])
+ service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = _make_opensearch_client()
+ sm = _make_session_manager(opensearch_client)
+
+ result = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["a", "b"],
+ )
+
+ assert result == []
+ opensearch_client.delete.assert_not_awaited()
+
+
+# ---------------------------------------------------------------------------
+# Happy path (DLS-safe enumerate-then-delete)
+# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_happy_path_deletes_orphans():
- """Indexed has [a, b, c]; remote has [a, c] -> orphan = [b]."""
+ """Indexed has [a, b, c]; remote has [a, c] → orphan = [b].
+
+ The implementation must use the DLS-safe pattern: enumerate visible chunk
+ _ids via search, then issue one primary-id delete per chunk. delete_by_query
+ must NEVER be called (silently no-opped under DLS).
+ """
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
connector = _make_connector(remote_file_ids=["a", "c"])
service = _make_service([conn], connector_lookup={"c1": connector})
- opensearch_client = AsyncMock()
- opensearch_client.delete_by_query.return_value = {"deleted": 7}
+ # One chunk belongs to orphan "b"
+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-0"])
sm = _make_session_manager(opensearch_client)
- deleted = await reconcile_orphans_for_connector_type(
+ result = await reconcile_orphans_for_connector_type(
connector_type="sharepoint",
user_id="alice",
connector_service=service,
@@ -184,25 +228,42 @@ async def test_happy_path_deletes_orphans():
existing_file_ids=["a", "b", "c"],
)
- assert deleted == 7
- opensearch_client.delete_by_query.assert_awaited_once()
- body = opensearch_client.delete_by_query.await_args.kwargs["body"]
- assert body == {"query": {"terms": {"document_id": ["b"]}}}
+ assert result == ["b"]
+
+ # DLS-safe: must NOT call delete_by_query
+ opensearch_client.delete_by_query.assert_not_called()
+
+ # Must enumerate via search first…
+ opensearch_client.search.assert_awaited_once()
+ search_body = opensearch_client.search.await_args.kwargs["body"]
+ assert search_body["query"] == {"terms": {"document_id": ["b"]}}
+
+ # …then delete each chunk by primary _id
+ opensearch_client.delete.assert_awaited_once()
+ delete_kwargs = opensearch_client.delete.await_args.kwargs
+ assert delete_kwargs["id"] == "chunk-b-0"
+ assert delete_kwargs.get("refresh") is True
@pytest.mark.asyncio
-async def test_no_orphans_skips_delete_call():
- """If every indexed ID still exists remotely, we must not issue an
- empty delete_by_query."""
+async def test_delete_failure_does_not_raise():
+ """If the bulk delete blows up, the helper must swallow it (no exception).
+ The function still returns the orphan IDs that were identified — callers
+ use the return value to know *what* was orphaned, not whether deletion
+ succeeded."""
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
- connector = _make_connector(remote_file_ids=["a", "b"])
+ connector = _make_connector(remote_file_ids=["a"])
service = _make_service([conn], connector_lookup={"c1": connector})
+
opensearch_client = AsyncMock()
+ # Make the search call itself fail so delete_chunks_by_document_ids raises
+ opensearch_client.search = AsyncMock(side_effect=RuntimeError("opensearch unavailable"))
sm = _make_session_manager(opensearch_client)
- deleted = await reconcile_orphans_for_connector_type(
+ # Must not raise even though deletion fails
+ result = await reconcile_orphans_for_connector_type(
connector_type="sharepoint",
user_id="alice",
connector_service=service,
@@ -211,16 +272,18 @@ async def test_no_orphans_skips_delete_call():
existing_file_ids=["a", "b"],
)
- assert deleted == 0
- opensearch_client.delete_by_query.assert_not_awaited()
+ # "b" was identified as orphaned; deletion failed silently
+ assert result == ["b"]
+
+
+# ---------------------------------------------------------------------------
+# Multi-connection isolation
+# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_multi_connection_union_preserves_files_present_in_any_connection():
- """Multi-connection isolation:
- - User has conn-A and conn-B (both SharePoint, different sites).
- - Indexed has [a, b]. conn-A has [a]. conn-B has [b].
- - Neither file is an orphan — both should be preserved."""
+ """conn-A has [a]. conn-B has [b]. Both [a, b] must be preserved."""
from api.connectors import reconcile_orphans_for_connector_type
conn_a = _make_connection("conn-a")
@@ -231,10 +294,10 @@ async def test_multi_connection_union_preserves_files_present_in_any_connection(
[conn_a, conn_b],
connector_lookup={"conn-a": connector_a, "conn-b": connector_b},
)
- opensearch_client = AsyncMock()
+ opensearch_client = _make_opensearch_client()
sm = _make_session_manager(opensearch_client)
- deleted = await reconcile_orphans_for_connector_type(
+ result = await reconcile_orphans_for_connector_type(
connector_type="sharepoint",
user_id="alice",
connector_service=service,
@@ -243,14 +306,13 @@ async def test_multi_connection_union_preserves_files_present_in_any_connection(
existing_file_ids=["a", "b"],
)
- assert deleted == 0
- opensearch_client.delete_by_query.assert_not_awaited()
+ assert result == []
+ opensearch_client.delete.assert_not_awaited()
@pytest.mark.asyncio
async def test_multi_connection_one_offline_aborts_even_if_other_succeeds():
- """If conn-A lists fine but conn-B is unauthenticated, we MUST NOT
- treat files only in conn-B as orphans. Strict gating bails out."""
+ """conn-B unauthenticated — must abort even though conn-A succeeded."""
from api.connectors import reconcile_orphans_for_connector_type
conn_a = _make_connection("conn-a")
@@ -261,27 +323,25 @@ async def test_multi_connection_one_offline_aborts_even_if_other_succeeds():
[conn_a, conn_b],
connector_lookup={"conn-a": connector_a, "conn-b": connector_b},
)
- opensearch_client = AsyncMock()
+ opensearch_client = _make_opensearch_client()
sm = _make_session_manager(opensearch_client)
- deleted = await reconcile_orphans_for_connector_type(
+ result = await reconcile_orphans_for_connector_type(
connector_type="sharepoint",
user_id="alice",
connector_service=service,
session_manager=sm,
jwt_token=None,
- existing_file_ids=["a", "b"], # b would look like an orphan if we trusted conn-A alone
+ existing_file_ids=["a", "b"],
)
- assert deleted == 0
- opensearch_client.delete_by_query.assert_not_awaited()
+ assert result == []
+ opensearch_client.delete.assert_not_awaited()
@pytest.mark.asyncio
async def test_paginated_listing_aggregates_all_pages():
- """Remote listings are paginated. The reconcile must walk every page
- before computing the orphan set, otherwise files on page 2 look like
- orphans."""
+ """Remote listings are paginated — must walk every page."""
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
@@ -289,15 +349,15 @@ async def test_paginated_listing_aggregates_all_pages():
connector.is_authenticated = True
pages = [
{"files": [{"id": "a"}], "nextPageToken": "tok-1"},
- {"files": [{"id": "b"}, {"id": "c"}]}, # last page, no token
+ {"files": [{"id": "b"}, {"id": "c"}]},
]
connector.list_files = AsyncMock(side_effect=pages)
service = _make_service([conn], connector_lookup={"c1": connector})
- opensearch_client = AsyncMock()
+ opensearch_client = _make_opensearch_client()
sm = _make_session_manager(opensearch_client)
- deleted = await reconcile_orphans_for_connector_type(
+ result = await reconcile_orphans_for_connector_type(
connector_type="sharepoint",
user_id="alice",
connector_service=service,
@@ -306,33 +366,79 @@ async def test_paginated_listing_aggregates_all_pages():
existing_file_ids=["a", "b", "c"],
)
- assert deleted == 0
+ assert result == []
assert connector.list_files.await_count == 2
- opensearch_client.delete_by_query.assert_not_awaited()
+ opensearch_client.delete.assert_not_awaited()
+
+
+# ---------------------------------------------------------------------------
+# id_field routing — connector_file_id vs document_id
+# ---------------------------------------------------------------------------
@pytest.mark.asyncio
-async def test_delete_failure_does_not_raise():
- """If the bulk delete itself blows up, the helper must swallow it and
- return 0. The caller (sync) should still proceed to re-sync surviving
- files — leaving orphans is recoverable, raising is not."""
+async def test_connector_file_id_field_used_when_specified():
+ """When id_field='connector_file_id', the delete query must target that
+ field. This is the non-Langflow (DISABLE_INGEST_WITH_LANGFLOW=True) path
+ where chunks carry document_id=SHA_hash but connector_file_id=connector_ID.
+ """
from api.connectors import reconcile_orphans_for_connector_type
conn = _make_connection("c1")
- connector = _make_connector(remote_file_ids=["a"])
+ # Remote only has "sp-guid-a"; "sp-guid-b" is orphaned
+ connector = _make_connector(remote_file_ids=["sp-guid-a"])
service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-b-0", "chunk-b-1"])
+ sm = _make_session_manager(opensearch_client)
- opensearch_client = AsyncMock()
- opensearch_client.delete_by_query.side_effect = RuntimeError("opensearch unavailable")
+ result = await reconcile_orphans_for_connector_type(
+ connector_type="sharepoint",
+ user_id="alice",
+ connector_service=service,
+ session_manager=sm,
+ jwt_token=None,
+ existing_file_ids=["sp-guid-a", "sp-guid-b"],
+ id_field="connector_file_id",
+ )
+
+ assert result == ["sp-guid-b"]
+
+ opensearch_client.delete_by_query.assert_not_called()
+
+ search_body = opensearch_client.search.await_args.kwargs["body"]
+ assert search_body["query"] == {"terms": {"connector_file_id": ["sp-guid-b"]}}, (
+ f"Expected connector_file_id query, got: {search_body['query']}"
+ )
+
+ assert opensearch_client.delete.await_count == 2
+
+
+async def test_document_id_field_used_by_default():
+ """Default id_field='document_id' preserves the Langflow path behavior
+ where document_id already holds the connector source ID.
+ """
+ from api.connectors import reconcile_orphans_for_connector_type
+
+ conn = _make_connection("c1")
+ connector = _make_connector(remote_file_ids=["lf-id-a"])
+ service = _make_service([conn], connector_lookup={"c1": connector})
+ opensearch_client = _make_opensearch_client(chunk_ids=["chunk-lf-b-0"])
sm = _make_session_manager(opensearch_client)
- deleted = await reconcile_orphans_for_connector_type(
+ result = await reconcile_orphans_for_connector_type(
connector_type="sharepoint",
user_id="alice",
connector_service=service,
session_manager=sm,
jwt_token=None,
- existing_file_ids=["a", "b"],
+ existing_file_ids=["lf-id-a", "lf-id-b"],
+ # id_field defaults to "document_id"
)
- assert deleted == 0
+ assert result == ["lf-id-b"]
+
+ search_body = opensearch_client.search.await_args.kwargs["body"]
+ assert search_body["query"] == {"terms": {"document_id": ["lf-id-b"]}}, (
+ f"Expected document_id query, got: {search_body['query']}"
+ )
diff --git a/tests/unit/connectors/test_google_drive_orphan_detection.py b/tests/unit/connectors/test_google_drive_orphan_detection.py
new file mode 100644
index 000000000..7c98e6144
--- /dev/null
+++ b/tests/unit/connectors/test_google_drive_orphan_detection.py
@@ -0,0 +1,144 @@
+"""Google Drive orphan-detection correctness.
+
+Pins two invariants introduced to fix Google Drive sync:
+
+1. Trashed files must be treated as missing in `_iter_selected_items` when
+ iterating the `file_ids` path (the path used by `compute_orphans_for_
+ connector_type`). Files moved to Google Drive trash still exist by ID and
+ `_get_file_meta_by_id` returns their metadata — without an explicit trash
+ check the orphan-detection would consider them still present and fail to
+ surface them as orphans.
+
+2. `authenticate()` must not make a live API call. The sanity-check
+ (`files().get("root")`) was causing false "connection may need
+ re-authentication" warnings when the call failed transiently (rate-limit,
+ network blip) even though the OAuth token was valid. Credential validity
+ is already ensured by `load_credentials()` and `oauth.is_authenticated()`.
+"""
+
+import sys
+import threading
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+def _make_connector(file_ids: list[str]):
+ """Build a minimal GoogleDriveConnector via __new__, bypassing __init__.
+
+ Only the attributes read by `_iter_selected_items` and `authenticate` are
+ set. This avoids the need for real OAuth credentials or token files.
+ """
+ from connectors.google_drive.connector import GoogleDriveConfig, GoogleDriveConnector
+
+ connector = GoogleDriveConnector.__new__(GoogleDriveConnector)
+ connector.cfg = GoogleDriveConfig(
+ client_id="fake-client-id",
+ client_secret="fake-client-secret",
+ token_file="/tmp/fake-token.json",
+ file_ids=file_ids,
+ folder_ids=None,
+ include_mime_types=None,
+ exclude_mime_types=None,
+ )
+ connector.service = MagicMock()
+ connector._authenticated = False
+ connector._lock = threading.Lock()
+ connector._shortcut_cache = {}
+ return connector
+
+
+# ---------------------------------------------------------------------------
+# _iter_selected_items — trashed / missing / present
+# ---------------------------------------------------------------------------
+
+
+def test_trashed_file_not_in_iter_selected_items():
+ """A file that exists in Drive trash must be excluded from orphan listing.
+
+ Without this guard, `_get_file_meta_by_id` returns metadata for trashed
+ files (they still exist by ID), causing them to appear in `remote_ids`
+ and thus NOT be flagged as orphans.
+ """
+ connector = _make_connector(file_ids=["trashed-file-id"])
+
+ with patch.object(
+ connector,
+ "_get_file_meta_by_id",
+ return_value={"id": "trashed-file-id", "mimeType": "application/pdf", "trashed": True},
+ ):
+ result = connector._iter_selected_items()
+
+ assert result == [], "trashed file must not appear in iter_selected_items output"
+
+
+def test_non_trashed_file_is_in_iter_selected_items():
+ """A file that exists and is not trashed must appear in the results."""
+ connector = _make_connector(file_ids=["live-file-id"])
+
+ with patch.object(
+ connector,
+ "_get_file_meta_by_id",
+ return_value={"id": "live-file-id", "mimeType": "application/pdf", "trashed": False},
+ ):
+ result = connector._iter_selected_items()
+
+ assert len(result) == 1
+ assert result[0]["id"] == "live-file-id"
+
+
+def test_missing_file_id_not_in_iter_selected_items():
+ """A file that returns None from `_get_file_meta_by_id` (e.g. permanently
+ deleted or permission error) must not appear in the results.
+
+ Regression guard for the pre-existing `if not meta: continue` behavior.
+ """
+ connector = _make_connector(file_ids=["gone-file-id"])
+
+ with patch.object(connector, "_get_file_meta_by_id", return_value=None):
+ result = connector._iter_selected_items()
+
+ assert result == []
+
+
+# ---------------------------------------------------------------------------
+# authenticate() — no live API call
+# ---------------------------------------------------------------------------
+
+
+async def test_authenticate_succeeds_without_live_api_call():
+ """authenticate() must not make a live API call to verify credentials.
+
+ The sanity-check `files().get("root").execute()` caused false auth
+ failures when the call failed transiently. Removing it means successful
+ authentication is determined solely by credential validity — which
+ `load_credentials()` and `oauth.is_authenticated()` already ensure.
+ """
+ from connectors.google_drive.connector import GoogleDriveConnector
+
+ connector = GoogleDriveConnector.__new__(GoogleDriveConnector)
+ connector._authenticated = False
+
+ fake_creds = MagicMock()
+ fake_service = MagicMock()
+
+ connector.oauth = MagicMock()
+ connector.oauth.load_credentials = AsyncMock(return_value=fake_creds)
+ connector.oauth.is_authenticated = AsyncMock(return_value=True)
+ connector.oauth.get_service = MagicMock(return_value=fake_service)
+
+ result = await connector.authenticate()
+
+ assert result is True
+ assert connector._authenticated is True
+ # The service must be set — connector is usable after authenticate().
+ assert connector.service is fake_service
+ # No live API call must have been made.
+ fake_service.files.assert_not_called()
diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.py
index 9cdfe1685..a866939e5 100644
--- a/tests/unit/test_connector_processor_filename_dedupe.py
+++ b/tests/unit/test_connector_processor_filename_dedupe.py
@@ -131,8 +131,9 @@ async def test_connector_processor_deletes_then_ingests_when_replace_true():
@pytest.mark.asyncio
async def test_connector_processor_deletes_chunks_when_source_returns_404():
"""When the source connector reports the file is gone (404), the processor
- must remove the already-indexed chunks for that document_id. Regression
- test for SharePoint sync leaving orphan chunks after a source-side delete.
+ must remove the already-indexed chunks queried by connector_file_id — NOT
+ document_id. Chunks are indexed with document_id=file_hash (SHA of content)
+ which differs from file_id, so querying document_id would find nothing.
"""
processor = _build_connector_processor(replace_duplicates=False)
@@ -168,6 +169,16 @@ async def test_connector_processor_deletes_chunks_when_source_returns_404():
assert upload_task.successful_files == 1
opensearch_client.delete.assert_awaited_once()
+ # The search must target connector_file_id, not document_id.
+ # document_id holds a SHA content hash that won't match the connector file ID.
+ search_call = opensearch_client.search.await_args
+ query = search_call.kwargs["body"]["query"]
+ assert "terms" in query, f"expected a terms query, got: {query}"
+ assert "connector_file_id" in query["terms"], (
+ f"cleanup must query connector_file_id, not document_id. Query: {query}"
+ )
+ assert query["terms"]["connector_file_id"] == ["file-id-1"]
+
@pytest.mark.asyncio
async def test_connector_processor_proceeds_when_filename_absent():
diff --git a/tests/unit/test_processors_clear_stale_chunks.py b/tests/unit/test_processors_clear_stale_chunks.py
index 5c0246525..6136e8df0 100644
--- a/tests/unit/test_processors_clear_stale_chunks.py
+++ b/tests/unit/test_processors_clear_stale_chunks.py
@@ -217,3 +217,74 @@ async def test_delete_failure_does_not_abort_reindex(monkeypatch):
assert result["status"] == "indexed"
assert len(index_calls) == 2, "indexing must still happen if the pre-delete fails"
+
+
+async def test_connector_file_id_stored_in_chunk_when_provided(monkeypatch):
+ """When connector_file_id is passed to process_document_standard, every
+ indexed chunk must carry that value so the 404 cleanup path can find and
+ delete chunks by their connector source ID."""
+ processor, opensearch_client = _make_processor_with_mocks()
+ _patch_embedding_pipeline(monkeypatch, chunk_count=2)
+
+ opensearch_client.search = AsyncMock(return_value={"_scroll_id": None, "hits": {"hits": []}})
+ opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})
+ indexed_bodies: list[dict] = []
+ opensearch_client.index = AsyncMock(
+ side_effect=lambda **kw: indexed_bodies.append(kw.get("body", {}))
+ )
+
+ with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:
+ tmp.write(b"hello world")
+ tmp_path = tmp.name
+
+ try:
+ result = await processor.process_document_standard(
+ file_path=tmp_path,
+ file_hash="sha-abc",
+ owner_user_id="alice",
+ original_filename="report.txt",
+ connector_type="sharepoint",
+ connector_file_id="sharepoint-item-xyz",
+ )
+ finally:
+ Path(tmp_path).unlink(missing_ok=True)
+
+ assert result["status"] == "indexed"
+ assert len(indexed_bodies) == 2, "expected one index call per chunk"
+ for body in indexed_bodies:
+ assert body.get("connector_file_id") == "sharepoint-item-xyz", (
+ f"connector_file_id missing or wrong in chunk: {body}"
+ )
+ assert body.get("document_id") == "sha-abc", "document_id must still be the content hash"
+
+
+async def test_connector_file_id_absent_when_not_provided(monkeypatch):
+ """When connector_file_id is not passed (local uploads, non-connector paths),
+ the field must be absent from indexed chunks — no None/empty pollution."""
+ processor, opensearch_client = _make_processor_with_mocks()
+ _patch_embedding_pipeline(monkeypatch, chunk_count=1)
+
+ opensearch_client.search = AsyncMock(return_value={"_scroll_id": None, "hits": {"hits": []}})
+ opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})
+ indexed_bodies: list[dict] = []
+ opensearch_client.index = AsyncMock(
+ side_effect=lambda **kw: indexed_bodies.append(kw.get("body", {}))
+ )
+
+ with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:
+ tmp.write(b"hello world")
+ tmp_path = tmp.name
+
+ try:
+ await processor.process_document_standard(
+ file_path=tmp_path,
+ file_hash="sha-xyz",
+ owner_user_id="alice",
+ )
+ finally:
+ Path(tmp_path).unlink(missing_ok=True)
+
+ assert len(indexed_bodies) == 1
+ assert "connector_file_id" not in indexed_bodies[0]