OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: Enum IDs and delete OpenSearch docs by _id (#1638)
Edwin Josed564ab48ea4c
diff --git a/src/api/connectors.py b/src/api/connectors.pyindex 3325a5c89..ea31c828d 100644--- a/src/api/connectors.py+++ b/src/api/connectors.py@@ -165,16 +165,43 @@ async def compute_orphans_for_connector_type(connection_id=conn.connection_id,)return None- page_token = None- while True:- page = await connector.list_files(page_token=page_token)- for f in page.get("files", []):- fid = f.get("id")- if fid:- remote_ids.add(fid)- page_token = page.get("nextPageToken") or page.get("next_page_token")- if not page_token:- break++ # Drive the per-id existence check via cfg.file_ids when the+ # connector supports it (SharePoint / OneDrive / Google Drive).+ # The flat default of list_files() only returns the *root* listing+ # (e.g. /drive/root/children for SharePoint, files-only, no folder+ # traversal), so any folder-internal file in OpenSearch would be+ # absent from remote_ids and wrongly flagged as an orphan.+ # _list_selected_files iterates each id via _get_file_metadata_by_id+ # and silently drops missing ids, so the resulting `remote_ids` is+ # exactly "the subset of existing_file_ids that still exists at+ # source" — which is what orphan detection actually needs.+ cfg = getattr(connector, "cfg", None)+ scoped_listing = cfg is not None and bool(existing_file_ids)++ original_file_ids = None+ original_folder_ids = None+ if scoped_listing:+ original_file_ids = getattr(cfg, "file_ids", None)+ original_folder_ids = getattr(cfg, "folder_ids", None)+ cfg.file_ids = list(existing_file_ids)+ cfg.folder_ids = None++ try:+ page_token = None+ while True:+ page = await connector.list_files(page_token=page_token)+ for f in page.get("files", []):+ fid = f.get("id")+ if fid:+ remote_ids.add(fid)+ page_token = page.get("nextPageToken") or page.get("next_page_token")+ if not page_token:+ break+ finally:+ if scoped_listing:+ cfg.file_ids = original_file_ids+ cfg.folder_ids = original_folder_idsexcept Exception as e:logger.warning("Skipping orphan compute — listing failed",diff --git a/src/api/documents.py b/src/api/documents.pyindex 421f966fe..3a7254930 100644--- a/src/api/documents.py+++ b/src/api/documents.py@@ -127,12 +127,28 @@ async def delete_chunks_by_document_ids(opensearch_client,index_name: str,) -> int:- """Bulk delete OpenSearch chunks by document_id. Returns deleted count."""+ """Bulk delete OpenSearch chunks by document_id. Returns deleted count.++ DLS-safe: enumerate the visible chunk _ids via search, then issue a single+ delete per primary id. `delete_by_query` is silently no-opped under DLS+ (returns deleted:N but leaves docs in place — confirmed in the SharePoint+ rename repro debug log).+ """if not document_ids:return 0- body = {"query": {"terms": {"document_id": document_ids}}}- res = await opensearch_client.delete_by_query(index=index_name, body=body, conflicts="proceed")- return res.get("deleted", 0)+ from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids++ chunk_ids = await collect_visible_document_ids(+ opensearch_client,+ index=index_name,+ query={"terms": {"document_id": document_ids}},+ )+ return await delete_document_ids(+ opensearch_client,+ index=index_name,+ document_ids=chunk_ids,+ refresh=True,+ )async def _ensure_index_exists(jwt_token: str = None):diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.pyindex 45e7d6314..5cd2716de 100644--- a/src/connectors/langflow_connector_service.py+++ b/src/connectors/langflow_connector_service.py@@ -1,13 +1,13 @@-from typing import Any, Dict, List, Optional+from typing import Any# Create custom processor for connector files using Langflowfrom models.processors import LangflowConnectorFileProcessorfrom services.langflow_file_service import LangflowFileService+from utils.file_utils import clean_connector_filename, get_file_extensionfrom utils.logging_config import get_loggerfrom .base import BaseConnector, ConnectorDocumentfrom .connection_manager import ConnectionManager-from utils.file_utils import get_file_extension, clean_connector_filenamelogger = get_logger(__name__)@@ -34,7 +34,7 @@ class LangflowConnectorService:"""Initialize the service by loading existing connections"""await self.connection_manager.load_connections()- async def get_connector(self, connection_id: str) -> Optional[BaseConnector]:+ async def get_connector(self, connection_id: str) -> BaseConnector | None:"""Get a connector by connection ID"""return await self.connection_manager.get_connector(connection_id)@@ -46,8 +46,8 @@ class LangflowConnectorService:jwt_token: str = None,owner_name: str = None,owner_email: str = None,- ingest_settings: Optional[Dict[str, Any]] = None,- ) -> Dict[str, Any]:+ ingest_settings: dict[str, Any] | None = None,+ ) -> dict[str, Any]:"""Process a document from a connector using LangflowFileService pattern"""logger.debug(@@ -69,7 +69,7 @@ class LangflowConnectorService:# Step 1: Upload file to Langflowlogger.debug("Uploading file to Langflow", filename=document.filename)content = document.content-+# Clean filename and ensure we don't add a double extensionprocessed_filename = clean_connector_filename(document.filename, document.mimetype)@@ -79,24 +79,53 @@ class LangflowConnectorService:document.mimetype or "application/octet-stream",)- # Step 0: Delete existing chunks for this file before re-ingesting- # This prevents duplicate chunks when syncing files+ # Step 0: Delete existing chunks for this file before re-ingesting.+ # Match on document_id (the stable connector item ID, e.g. the+ # SharePoint Graph item id) — NOT on filename. On a rename, the+ # `processed_filename` here is the NEW name while OpenSearch chunks+ # still carry the OLD name, so a filename-keyed delete misses them+ # and the re-ingest leaves duplicate chunks (same document_id, two+ # different filenames). Also use enumerate-then-delete-by-id rather+ # than delete_by_query, which is silently no-opped under DLS.if self.session_manager:+ from config.settings import get_index_name+ from utils.opensearch_delete import (+ collect_visible_document_ids,+ delete_document_ids,+ )++ opensearch_client = self.session_manager.get_user_opensearch_client(+ owner_user_id, jwt_token+ )try:- from config.settings import get_index_name- opensearch_client = self.session_manager.get_user_opensearch_client(owner_user_id, jwt_token)- delete_body = {"query": {"term": {"filename": processed_filename}}}- delete_result = await opensearch_client.delete_by_query(index=get_index_name(), body=delete_body)- deleted_count = delete_result.get("deleted", 0)- logger.info("Deleted existing chunks before re-ingestion", filename=processed_filename, deleted_count=deleted_count)+ chunk_ids = await collect_visible_document_ids(+ opensearch_client,+ index=get_index_name(),+ query={"term": {"document_id": document.id}},+ )+ deleted_count = await delete_document_ids(+ opensearch_client,+ index=get_index_name(),+ document_ids=chunk_ids,+ refresh=True,+ )+ logger.info(+ "Deleted existing chunks before re-ingestion",+ document_id=document.id,+ filename=processed_filename,+ deleted_count=deleted_count,+ )except Exception as delete_err:- logger.warning("Failed to delete existing chunks before re-ingestion", filename=processed_filename, error=str(delete_err))+ logger.warning(+ "Failed to delete existing chunks before re-ingestion",+ document_id=document.id,+ filename=processed_filename,+ error=str(delete_err),+ )langflow_file_id = None # Initialize to track if upload succeededtry:- upload_result = await self.langflow_service.upload_user_file(- file_tuple, jwt_token- )+ upload_result = await self.langflow_service.upload_user_file(file_tuple, jwt_token)langflow_file_id = upload_result["id"]langflow_file_path = upload_result["path"]@@ -107,9 +136,7 @@ class LangflowConnectorService:)# Step 2: Run ingestion flow with the uploaded file- logger.debug(- "Running Langflow ingestion flow", file_path=langflow_file_path- )+ logger.debug("Running Langflow ingestion flow", file_path=langflow_file_path)connector_tweak_settings = Noneif isinstance(ingest_settings, dict):@@ -192,7 +219,6 @@ class LangflowConnectorService:)raise-async def sync_connector_files(self,connection_id: str,@@ -214,9 +240,7 @@ class LangflowConnectorService:connector = await self.get_connector(connection_id)if not connector:- raise ValueError(- f"Connection '{connection_id}' not found or not authenticated"- )+ raise ValueError(f"Connection '{connection_id}' not found or not authenticated")logger.debug("Got connector", authenticated=connector.is_authenticated)@@ -224,7 +248,7 @@ class LangflowConnectorService:raise ValueError(f"Connection '{connection_id}' not authenticated")# Collect files to process (limited by max_files)- files_to_process = []+ files_to_process: list[dict[str, Any]] = []page_token = None# Calculate page size to minimize API calls@@ -232,13 +256,9 @@ class LangflowConnectorService:while True:# List files from connector with limit- logger.debug(- "Calling list_files", page_size=page_size, page_token=page_token- )- file_list = await connector.list_files(page_token, limit=page_size)- logger.debug(- "Got files from connector", file_count=len(file_list.get("files", []))- )+ logger.debug("Calling list_files", page_size=page_size, page_token=page_token)+ file_list = await connector.list_files(page_token, max_files=page_size)+ logger.debug("Got files from connector", file_count=len(file_list.get("files", [])))files = file_list["files"]if not files:@@ -293,15 +313,15 @@ class LangflowConnectorService:self,connection_id: str,user_id: str,- file_ids: List[str],+ file_ids: list[str],jwt_token: str = None,- file_infos: List[Dict[str, Any]] = None,- ingest_settings: Optional[Dict[str, Any]] = None,+ file_infos: list[dict[str, Any]] = None,+ ingest_settings: dict[str, Any] | None = None,) -> str:"""Sync specific files by their IDs using Langflow processing.Automatically expands folders to their contents.-+Args:connection_id: The connection IDuser_id: The user ID@@ -317,9 +337,7 @@ class LangflowConnectorService:connector = await self.get_connector(connection_id)if not connector:- raise ValueError(- f"Connection '{connection_id}' not found or not authenticated"- )+ raise ValueError(f"Connection '{connection_id}' not found or not authenticated")if not connector.is_authenticated:raise ValueError(f"Connection '{connection_id}' not authenticated")@@ -334,7 +352,7 @@ class LangflowConnectorService:# If file_infos provided, cache them in the connector for later use# This allows get_file_content to use download URLs directly- if file_infos and hasattr(connector, 'set_file_infos'):+ if file_infos and hasattr(connector, "set_file_infos"):connector.set_file_infos(file_infos)logger.info(f"Cached {len(file_infos)} file infos with download URLs in connector")@@ -357,8 +375,8 @@ class LangflowConnectorService:# carefully selected IDs passed in.if cfg is not None:try:- cfg.file_ids = file_ids # type: ignore- cfg.folder_ids = None # type: ignore+ cfg.file_ids = file_ids+ cfg.folder_ids = None# Expand file IDs — folders become their individual file contentsresult = await connector.list_files()@@ -383,8 +401,8 @@ class LangflowConnectorService:# Fallback to original file_ids if expansion failsexpanded_file_ids = file_idsfinally:- cfg.file_ids = original_file_ids # type: ignore- cfg.folder_ids = original_folder_ids # type: ignore+ cfg.file_ids = original_file_ids+ cfg.folder_ids = original_folder_idsprocessor = LangflowConnectorFileProcessor(self,@@ -401,9 +419,7 @@ class LangflowConnectorService:original_filenames = {}if file_infos:original_filenames = {- f["id"]: clean_connector_filename(- f["name"], f.get("mimeType") or f.get("mimetype")- )+ f["id"]: clean_connector_filename(f["name"], f.get("mimeType") or f.get("mimetype"))for f in file_infosif "id" in f and "name" in f}@@ -414,6 +430,6 @@ class LangflowConnectorService:return task_id- async def _get_connector(self, connection_id: str) -> Optional[BaseConnector]:+ async def _get_connector(self, connection_id: str) -> BaseConnector | None:"""Get a connector by connection ID (alias for get_connector)"""return await self.get_connector(connection_id)diff --git a/src/models/processors.py b/src/models/processors.pyindex 01df83a88..02771678a 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -303,6 +303,36 @@ class TaskProcessor:opensearch_client, embedding_model, get_index_name(), dimensions)+ # Clear stale chunks from a prior indexing of this document. Chunks are+ # stored under ids {file_hash}_{i}; if the new chunk count is lower than+ # the prior one, trailing chunks (e.g. with an old filename after a+ # SharePoint rename) would otherwise survive the per-chunk upsert.+ # DLS-safe: enumerate then delete by primary _id (delete_by_query is+ # silently filtered under DLS).+ try:+ from utils.opensearch_delete import (+ collect_visible_document_ids,+ delete_document_ids,+ )++ stale_chunk_ids = await collect_visible_document_ids(+ opensearch_client,+ index=get_index_name(),+ query={"term": {"document_id": file_hash}},+ )+ await delete_document_ids(+ opensearch_client,+ index=get_index_name(),+ document_ids=stale_chunk_ids,+ refresh=True,+ )+ except Exception as e:+ logger.warning(+ "Failed to clear stale chunks before re-index; proceeding",+ file_hash=file_hash,+ error=str(e),+ )+# Index each chunk as a separate documentfor i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings, strict=True)):chunk_doc = {diff --git a/src/utils/opensearch_delete.py b/src/utils/opensearch_delete.pynew file mode 100644index 000000000..4e9db031f--- /dev/null+++ b/src/utils/opensearch_delete.py@@ -0,0 +1,75 @@+from collections.abc import Iterable++from opensearchpy.exceptions import NotFoundError++from utils.logging_config import get_logger++logger = get_logger(__name__)+++async def collect_visible_document_ids(+ opensearch_client,+ *,+ index: str,+ query: dict,+ source: bool | list[str] = False,+ page_size: int = 1000,+ scroll_ttl: str = "2m",+) -> list[str]:+ """Collect visible document IDs for a query using the caller's OpenSearch client."""+ search_body = {"query": query, "size": page_size, "_source": source}+ response = await opensearch_client.search(+ index=index,+ body=search_body,+ scroll=scroll_ttl,+ )+ scroll_id = response.get("_scroll_id")+ document_ids: list[str] = []++ try:+ while True:+ hits = response.get("hits", {}).get("hits", [])+ document_ids.extend(hit["_id"] for hit in hits if hit.get("_id"))+ if not hits or len(hits) < page_size or not scroll_id:+ break+ response = await opensearch_client.scroll(scroll_id=scroll_id, scroll=scroll_ttl)+ scroll_id = response.get("_scroll_id") or scroll_id+ finally:+ if scroll_id and hasattr(opensearch_client, "clear_scroll"):+ try:+ await opensearch_client.clear_scroll(scroll_id=scroll_id)+ except Exception as e:+ logger.debug("Failed to clear OpenSearch scroll context", error=str(e))++ return document_ids+++async def delete_document_ids(+ opensearch_client,+ *,+ index: str,+ document_ids: Iterable[str],+ refresh: bool = True,+) -> int:+ """Delete concrete OpenSearch document IDs through the caller's client.++ delete_by_query is silently no-opped under DLS / certain security plugins+ (returns deleted:N but leaves the docs in place). Single deletes keyed on+ the primary _id are reliable, so enumerate visible IDs first and then issue+ a delete per ID.+ """+ deleted_count = 0+ for document_id in document_ids:+ try:+ result = await opensearch_client.delete(+ index=index,+ id=document_id,+ refresh=refresh,+ )+ except NotFoundError:+ continue++ if result.get("result") == "deleted":+ deleted_count += 1++ return deleted_countdiff --git a/tests/unit/api/test_delete_chunks_by_document_ids.py b/tests/unit/api/test_delete_chunks_by_document_ids.pyindex 005a1477d..5ae43fc49 100644--- a/tests/unit/api/test_delete_chunks_by_document_ids.py+++ b/tests/unit/api/test_delete_chunks_by_document_ids.py@@ -2,8 +2,9 @@Pins the contract of `src/api/documents.py::delete_chunks_by_document_ids`:- empty input is a no-op (no OpenSearch call)-- non-empty input issues a single delete_by_query with terms(document_id, ...)-- returns the count reported by OpenSearch+- non-empty input enumerates visible chunk _ids and deletes them one-by-one+ by primary id (DLS-safe; `delete_by_query` is silently filtered under DLS)+- returns the count of successful single deletes"""import sys@@ -26,37 +27,61 @@ async def test_empty_ids_short_circuits_without_calling_opensearch():deleted = await delete_chunks_by_document_ids([], opensearch_client, "test-index")assert deleted == 0+ opensearch_client.search.assert_not_awaited()+ opensearch_client.delete.assert_not_awaited()opensearch_client.delete_by_query.assert_not_awaited()@pytest.mark.asyncio-async def test_issues_single_delete_by_query_with_terms_filter():+async def test_deletes_each_visible_chunk_id_by_primary_id():+ """The helper must enumerate visible chunk _ids and issue a primary-id+ delete for each. delete_by_query is forbidden because DLS / certain+ security plugins silently no-op it (returns deleted:N but leaves docs)."""from api.documents import delete_chunks_by_document_idsopensearch_client = AsyncMock()- opensearch_client.delete_by_query.return_value = {"deleted": 12}+ # Visible chunk _ids for the requested document_ids.+ chunk_ids = ["chunk-1", "chunk-2", "chunk-3"]+ opensearch_client.search.return_value = {+ "_scroll_id": None,+ "hits": {"hits": [{"_id": cid} for cid in chunk_ids]},+ }+ opensearch_client.delete.return_value = {"result": "deleted"}- ids = ["abc", "def", "ghi"]- deleted = await delete_chunks_by_document_ids(ids, opensearch_client, "test-index")+ deleted = await delete_chunks_by_document_ids(+ ["doc-a", "doc-b"], opensearch_client, "test-index"+ )- assert deleted == 12- opensearch_client.delete_by_query.assert_awaited_once()- call = opensearch_client.delete_by_query.await_args- assert call.kwargs["index"] == "test-index"- assert call.kwargs["body"] == {"query": {"terms": {"document_id": ids}}}- # conflicts="proceed" so a concurrent reindex doesn't abort the bulk delete.- assert call.kwargs.get("conflicts") == "proceed"+ assert deleted == len(chunk_ids)++ # delete_by_query must NOT be used — silently filtered under DLS.+ opensearch_client.delete_by_query.assert_not_awaited()++ # Search query must target the document_id field with terms(...).+ search_call = opensearch_client.search.await_args+ assert search_call.kwargs["index"] == "test-index"+ assert search_call.kwargs["body"]["query"] == {"terms": {"document_id": ["doc-a", "doc-b"]}}++ # One primary-id delete per visible chunk, refresh=True so the delete is+ # immediately visible to the re-index that typically follows.+ delete_calls = opensearch_client.delete.await_args_list+ assert len(delete_calls) == len(chunk_ids)+ for call, cid in zip(delete_calls, chunk_ids, strict=True):+ assert call.kwargs["index"] == "test-index"+ assert call.kwargs["id"] == cid+ assert call.kwargs.get("refresh") is True@pytest.mark.asyncio-async def test_returns_zero_when_response_missing_deleted_field():- """Defensive: if OpenSearch returns an unexpected payload shape, we- should not crash — we surface 0 deletions."""+async def test_returns_zero_when_no_visible_chunks_match():+ """When the search returns no hits, no deletes are issued and the count+ is zero. This is the steady-state path (nothing to clean up)."""from api.documents import delete_chunks_by_document_idsopensearch_client = AsyncMock()- opensearch_client.delete_by_query.return_value = {} # no "deleted" key+ opensearch_client.search.return_value = {"_scroll_id": None, "hits": {"hits": []}}deleted = await delete_chunks_by_document_ids(["abc"], opensearch_client, "test-index")assert deleted == 0+ opensearch_client.delete.assert_not_awaited()diff --git a/tests/unit/test_processors_clear_stale_chunks.py b/tests/unit/test_processors_clear_stale_chunks.pynew file mode 100644index 000000000..5c0246525--- /dev/null+++ b/tests/unit/test_processors_clear_stale_chunks.py@@ -0,0 +1,219 @@+"""Re-indexing in `process_document_standard` must clear prior chunks for the+same `document_id` before writing the new ones.++Per-chunk indexing uses ids `{file_hash}_{i}` with `opensearch_client.index`+(upsert). Without a pre-delete, a re-index that produces fewer chunks than the+prior pass leaves trailing chunks `{file_hash}_{N..M-1}` behind with the OLD+metadata — most visibly, the old filename after a SharePoint rename. This test+pins the invariant introduced to fix that.++Pins: `src/models/processors.py` :: TaskProcessor.process_document_standard.+"""++import sys+import tempfile+from pathlib import Path+from unittest.mock import AsyncMock, MagicMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++def _make_processor_with_mocks():+ """Build a TaskProcessor wired to mocks for every external dependency+ `process_document_standard` reaches. Returns (processor, opensearch_client)."""+ from models.processors import TaskProcessor++ opensearch_client = AsyncMock()+ # exists() is checked at the top of process_document_standard; returning+ # False forces the re-index path (the path where delete_by_query must fire).+ opensearch_client.exists = AsyncMock(return_value=False)++ session_manager = MagicMock()+ session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)++ document_service = MagicMock()+ document_service.session_manager = session_manager++ models_service = MagicMock()+ models_service.get_litellm_model_name = AsyncMock(return_value="text-embedding-3-small")++ docling_service = MagicMock() # unused for .txt path++ processor = TaskProcessor(+ document_service=document_service,+ models_service=models_service,+ docling_service=docling_service,+ )+ return processor, opensearch_client+++def _patch_embedding_pipeline(monkeypatch, chunk_count: int):+ """Stub out the docling / embedding / index-mapping side of+ process_document_standard so the test focuses on the OpenSearch write+ ordering. `chunk_count` controls how many chunks the simulated text-file+ parse produces.+ """+ from models import processors as processors_mod++ fake_slim_doc = {+ "id": "doc",+ "filename": "ignored.txt",+ "mimetype": "text/plain",+ "chunks": [{"page": 1, "text": f"chunk-{i}"} for i in range(chunk_count)],+ }+ monkeypatch.setattr(processors_mod, "process_text_file", lambda _path: fake_slim_doc)++ # Embedding model resolution path (config + fallback).+ fake_config = MagicMock()+ fake_config.knowledge.embedding_model = "text-embedding-3-small"+ monkeypatch.setattr(processors_mod, "get_openrag_config", lambda: fake_config)+ monkeypatch.setattr(processors_mod, "get_embedding_model", lambda: "text-embedding-3-small")+ monkeypatch.setattr(processors_mod, "get_index_name", lambda: "test-index")++ # Field-mapping helper writes nothing useful for this test.+ async def _ensure_embedding_field_exists(_client, _model, _index, _dims):+ return "embedding_field"++ monkeypatch.setattr(+ processors_mod, "ensure_embedding_field_exists", _ensure_embedding_field_exists+ )++ # chunk_texts_for_embeddings is imported lazily inside the function from+ # services.document_service — patch it at its source.+ from services import document_service as ds_mod++ monkeypatch.setattr(+ ds_mod,+ "chunk_texts_for_embeddings",+ lambda texts, max_tokens=8000: [list(texts)],+ )++ # patched_embedding_client.embeddings.create — return one embedding per text.+ # `clients` is the singleton imported at module scope; replace it wholesale+ # (the real one's `patched_embedding_client` is a read-only @property).+ class _FakeEmbedResp:+ def __init__(self, n):+ self.data = [{"embedding": [0.1, 0.2, 0.3]} for _ in range(n)]++ fake_embed_client = MagicMock()+ fake_embed_client.embeddings.create = AsyncMock(+ side_effect=lambda model, input: _FakeEmbedResp(len(input))+ )+ fake_clients = MagicMock()+ fake_clients.patched_embedding_client = fake_embed_client+ monkeypatch.setattr(processors_mod, "clients", fake_clients)+++async def test_stale_chunks_cleared_before_reindex(monkeypatch):+ """Stale chunks must be cleared (via primary-id deletes) before any new+ `index()` call so prior chunks (e.g. with an old filename after a rename)+ cannot survive the per-chunk upsert.++ DLS-safe pattern: enumerate visible chunk _ids via search, then issue a+ `delete` per primary `_id`. `delete_by_query` is silently filtered under+ DLS and must NOT be used.+ """+ processor, opensearch_client = _make_processor_with_mocks()+ _patch_embedding_pipeline(monkeypatch, chunk_count=3)++ stale_chunk_ids = ["abc123_0", "abc123_1", "abc123_2", "abc123_3", "abc123_4"]+ op_order: list[tuple[str, dict]] = []++ async def _search(**kw):+ op_order.append(("search", kw))+ return {"_scroll_id": None, "hits": {"hits": [{"_id": cid} for cid in stale_chunk_ids]}}++ async def _delete(**kw):+ op_order.append(("delete", kw))+ return {"result": "deleted"}++ async def _index(**kw):+ op_order.append(("index", kw))++ opensearch_client.search = AsyncMock(side_effect=_search)+ opensearch_client.delete = AsyncMock(side_effect=_delete)+ opensearch_client.index = AsyncMock(side_effect=_index)++ with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:+ tmp.write(b"hello world")+ tmp_path = tmp.name++ try:+ await processor.process_document_standard(+ file_path=tmp_path,+ file_hash="abc123",+ owner_user_id="alice",+ original_filename="renamed.txt",+ connector_type="sharepoint",+ )+ finally:+ Path(tmp_path).unlink(missing_ok=True)++ ops = [op for op, _ in op_order]+ assert ops, "process_document_standard wrote nothing — fixture is broken"++ # 1) The enumerate-via-search runs first.+ assert ops[0] == "search", f"search must run before deletes. Saw: {ops}"+ search_kwargs = op_order[0][1]+ assert search_kwargs["body"]["query"] == {"term": {"document_id": "abc123"}}+ assert search_kwargs["index"] == "test-index"++ # 2) All deletes complete BEFORE any index() — they must precede re-indexing.+ delete_indices = [i for i, op in enumerate(ops) if op == "delete"]+ index_indices = [i for i, op in enumerate(ops) if op == "index"]+ assert delete_indices, "no delete was issued; stale chunks would survive"+ assert index_indices, "no chunks were indexed"+ assert max(delete_indices) < min(index_indices), (+ f"all deletes must complete before any index(). Saw order: {ops}"+ )++ # 3) One primary-id delete per visible stale chunk, refresh=True.+ delete_calls = [kw for op, kw in op_order if op == "delete"]+ assert len(delete_calls) == len(stale_chunk_ids)+ for call, expected_id in zip(delete_calls, stale_chunk_ids, strict=True):+ assert call["index"] == "test-index"+ assert call["id"] == expected_id+ assert call.get("refresh") is True++ # 4) delete_by_query must NEVER be used (DLS would silently filter it).+ if hasattr(opensearch_client, "delete_by_query"):+ opensearch_client.delete_by_query.assert_not_called()+++async def test_delete_failure_does_not_abort_reindex(monkeypatch):+ """A transient delete failure must be logged and swallowed — the per-chunk+ upsert still runs so the sync isn't worse off than today's behavior."""+ processor, opensearch_client = _make_processor_with_mocks()+ _patch_embedding_pipeline(monkeypatch, chunk_count=2)++ # Have the enumerate step itself blow up — that's the only "delete failure"+ # surface the helper exposes, since per-id delete swallows NotFoundError.+ opensearch_client.search = AsyncMock(side_effect=RuntimeError("os 503"))+ opensearch_client.delete = AsyncMock()+ index_calls: list[dict] = []+ opensearch_client.index = AsyncMock(side_effect=lambda **kw: index_calls.append(kw))++ with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:+ tmp.write(b"hello world")+ tmp_path = tmp.name++ try:+ result = await processor.process_document_standard(+ file_path=tmp_path,+ file_hash="abc123",+ owner_user_id="alice",+ original_filename="renamed.txt",+ connector_type="sharepoint",+ )+ finally:+ Path(tmp_path).unlink(missing_ok=True)++ assert result["status"] == "indexed"+ assert len(index_calls) == 2, "indexing must still happen if the pre-delete fails"