Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: Enum IDs and delete OpenSearch docs by _id (#1638)

Edwin Josed564ab48ea4c
diff --git a/src/api/connectors.py b/src/api/connectors.py
index 3325a5c89..ea31c828d 100644
--- a/src/api/connectors.py
+++ b/src/api/connectors.py
@@ -165,16 +165,43 @@ async def compute_orphans_for_connector_type(
connection_id=conn.connection_id,
)
return None
- page_token = None
- while True:
- page = await connector.list_files(page_token=page_token)
- for f in page.get("files", []):
- fid = f.get("id")
- if fid:
- remote_ids.add(fid)
- page_token = page.get("nextPageToken") or page.get("next_page_token")
- if not page_token:
- break
+
+ # Drive the per-id existence check via cfg.file_ids when the
+ # connector supports it (SharePoint / OneDrive / Google Drive).
+ # The flat default of list_files() only returns the *root* listing
+ # (e.g. /drive/root/children for SharePoint, files-only, no folder
+ # traversal), so any folder-internal file in OpenSearch would be
+ # absent from remote_ids and wrongly flagged as an orphan.
+ # _list_selected_files iterates each id via _get_file_metadata_by_id
+ # and silently drops missing ids, so the resulting `remote_ids` is
+ # exactly "the subset of existing_file_ids that still exists at
+ # source" — which is what orphan detection actually needs.
+ cfg = getattr(connector, "cfg", None)
+ scoped_listing = cfg is not None and bool(existing_file_ids)
+
+ original_file_ids = None
+ original_folder_ids = None
+ if scoped_listing:
+ original_file_ids = getattr(cfg, "file_ids", None)
+ original_folder_ids = getattr(cfg, "folder_ids", None)
+ cfg.file_ids = list(existing_file_ids)
+ cfg.folder_ids = None
+
+ try:
+ page_token = None
+ while True:
+ page = await connector.list_files(page_token=page_token)
+ for f in page.get("files", []):
+ fid = f.get("id")
+ if fid:
+ remote_ids.add(fid)
+ page_token = page.get("nextPageToken") or page.get("next_page_token")
+ if not page_token:
+ break
+ finally:
+ if scoped_listing:
+ cfg.file_ids = original_file_ids
+ cfg.folder_ids = original_folder_ids
except Exception as e:
logger.warning(
"Skipping orphan compute — listing failed",
diff --git a/src/api/documents.py b/src/api/documents.py
index 421f966fe..3a7254930 100644
--- a/src/api/documents.py
+++ b/src/api/documents.py
@@ -127,12 +127,28 @@ async def delete_chunks_by_document_ids(
opensearch_client,
index_name: str,
) -> int:
- """Bulk delete OpenSearch chunks by document_id. Returns deleted count."""
+ """Bulk delete OpenSearch chunks by document_id. Returns deleted count.
+
+ DLS-safe: enumerate the visible chunk _ids via search, then issue a single
+ delete per primary id. `delete_by_query` is silently no-opped under DLS
+ (returns deleted:N but leaves docs in place — confirmed in the SharePoint
+ rename repro debug log).
+ """
if not document_ids:
return 0
- body = {"query": {"terms": {"document_id": document_ids}}}
- res = await opensearch_client.delete_by_query(index=index_name, body=body, conflicts="proceed")
- return res.get("deleted", 0)
+ from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
+
+ chunk_ids = await collect_visible_document_ids(
+ opensearch_client,
+ index=index_name,
+ query={"terms": {"document_id": document_ids}},
+ )
+ return await delete_document_ids(
+ opensearch_client,
+ index=index_name,
+ document_ids=chunk_ids,
+ refresh=True,
+ )
async def _ensure_index_exists(jwt_token: str = None):
diff --git a/src/connectors/langflow_connector_service.py b/src/connectors/langflow_connector_service.py
index 45e7d6314..5cd2716de 100644
--- a/src/connectors/langflow_connector_service.py
+++ b/src/connectors/langflow_connector_service.py
@@ -1,13 +1,13 @@
-from typing import Any, Dict, List, Optional
+from typing import Any
# Create custom processor for connector files using Langflow
from models.processors import LangflowConnectorFileProcessor
from services.langflow_file_service import LangflowFileService
+from utils.file_utils import clean_connector_filename, get_file_extension
from utils.logging_config import get_logger
from .base import BaseConnector, ConnectorDocument
from .connection_manager import ConnectionManager
-from utils.file_utils import get_file_extension, clean_connector_filename
logger = get_logger(__name__)
@@ -34,7 +34,7 @@ class LangflowConnectorService:
"""Initialize the service by loading existing connections"""
await self.connection_manager.load_connections()
- async def get_connector(self, connection_id: str) -> Optional[BaseConnector]:
+ async def get_connector(self, connection_id: str) -> BaseConnector | None:
"""Get a connector by connection ID"""
return await self.connection_manager.get_connector(connection_id)
@@ -46,8 +46,8 @@ class LangflowConnectorService:
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
- ingest_settings: Optional[Dict[str, Any]] = None,
- ) -> Dict[str, Any]:
+ ingest_settings: dict[str, Any] | None = None,
+ ) -> dict[str, Any]:
"""Process a document from a connector using LangflowFileService pattern"""
logger.debug(
@@ -69,7 +69,7 @@ class LangflowConnectorService:
# Step 1: Upload file to Langflow
logger.debug("Uploading file to Langflow", filename=document.filename)
content = document.content
-
+
# Clean filename and ensure we don't add a double extension
processed_filename = clean_connector_filename(document.filename, document.mimetype)
@@ -79,24 +79,53 @@ class LangflowConnectorService:
document.mimetype or "application/octet-stream",
)
- # Step 0: Delete existing chunks for this file before re-ingesting
- # This prevents duplicate chunks when syncing files
+ # Step 0: Delete existing chunks for this file before re-ingesting.
+ # Match on document_id (the stable connector item ID, e.g. the
+ # SharePoint Graph item id) — NOT on filename. On a rename, the
+ # `processed_filename` here is the NEW name while OpenSearch chunks
+ # still carry the OLD name, so a filename-keyed delete misses them
+ # and the re-ingest leaves duplicate chunks (same document_id, two
+ # different filenames). Also use enumerate-then-delete-by-id rather
+ # than delete_by_query, which is silently no-opped under DLS.
if self.session_manager:
+ from config.settings import get_index_name
+ from utils.opensearch_delete import (
+ collect_visible_document_ids,
+ delete_document_ids,
+ )
+
+ opensearch_client = self.session_manager.get_user_opensearch_client(
+ owner_user_id, jwt_token
+ )
try:
- from config.settings import get_index_name
- opensearch_client = self.session_manager.get_user_opensearch_client(owner_user_id, jwt_token)
- delete_body = {"query": {"term": {"filename": processed_filename}}}
- delete_result = await opensearch_client.delete_by_query(index=get_index_name(), body=delete_body)
- deleted_count = delete_result.get("deleted", 0)
- logger.info("Deleted existing chunks before re-ingestion", filename=processed_filename, deleted_count=deleted_count)
+ chunk_ids = await collect_visible_document_ids(
+ opensearch_client,
+ index=get_index_name(),
+ query={"term": {"document_id": document.id}},
+ )
+ deleted_count = await delete_document_ids(
+ opensearch_client,
+ index=get_index_name(),
+ document_ids=chunk_ids,
+ refresh=True,
+ )
+ logger.info(
+ "Deleted existing chunks before re-ingestion",
+ document_id=document.id,
+ filename=processed_filename,
+ deleted_count=deleted_count,
+ )
except Exception as delete_err:
- logger.warning("Failed to delete existing chunks before re-ingestion", filename=processed_filename, error=str(delete_err))
+ logger.warning(
+ "Failed to delete existing chunks before re-ingestion",
+ document_id=document.id,
+ filename=processed_filename,
+ error=str(delete_err),
+ )
langflow_file_id = None # Initialize to track if upload succeeded
try:
- upload_result = await self.langflow_service.upload_user_file(
- file_tuple, jwt_token
- )
+ upload_result = await self.langflow_service.upload_user_file(file_tuple, jwt_token)
langflow_file_id = upload_result["id"]
langflow_file_path = upload_result["path"]
@@ -107,9 +136,7 @@ class LangflowConnectorService:
)
# Step 2: Run ingestion flow with the uploaded file
- logger.debug(
- "Running Langflow ingestion flow", file_path=langflow_file_path
- )
+ logger.debug("Running Langflow ingestion flow", file_path=langflow_file_path)
connector_tweak_settings = None
if isinstance(ingest_settings, dict):
@@ -192,7 +219,6 @@ class LangflowConnectorService:
)
raise
-
async def sync_connector_files(
self,
connection_id: str,
@@ -214,9 +240,7 @@ class LangflowConnectorService:
connector = await self.get_connector(connection_id)
if not connector:
- raise ValueError(
- f"Connection '{connection_id}' not found or not authenticated"
- )
+ raise ValueError(f"Connection '{connection_id}' not found or not authenticated")
logger.debug("Got connector", authenticated=connector.is_authenticated)
@@ -224,7 +248,7 @@ class LangflowConnectorService:
raise ValueError(f"Connection '{connection_id}' not authenticated")
# Collect files to process (limited by max_files)
- files_to_process = []
+ files_to_process: list[dict[str, Any]] = []
page_token = None
# Calculate page size to minimize API calls
@@ -232,13 +256,9 @@ class LangflowConnectorService:
while True:
# List files from connector with limit
- logger.debug(
- "Calling list_files", page_size=page_size, page_token=page_token
- )
- file_list = await connector.list_files(page_token, limit=page_size)
- logger.debug(
- "Got files from connector", file_count=len(file_list.get("files", []))
- )
+ logger.debug("Calling list_files", page_size=page_size, page_token=page_token)
+ file_list = await connector.list_files(page_token, max_files=page_size)
+ logger.debug("Got files from connector", file_count=len(file_list.get("files", [])))
files = file_list["files"]
if not files:
@@ -293,15 +313,15 @@ class LangflowConnectorService:
self,
connection_id: str,
user_id: str,
- file_ids: List[str],
+ file_ids: list[str],
jwt_token: str = None,
- file_infos: List[Dict[str, Any]] = None,
- ingest_settings: Optional[Dict[str, Any]] = None,
+ file_infos: list[dict[str, Any]] = None,
+ ingest_settings: dict[str, Any] | None = None,
) -> str:
"""
Sync specific files by their IDs using Langflow processing.
Automatically expands folders to their contents.
-
+
Args:
connection_id: The connection ID
user_id: The user ID
@@ -317,9 +337,7 @@ class LangflowConnectorService:
connector = await self.get_connector(connection_id)
if not connector:
- raise ValueError(
- f"Connection '{connection_id}' not found or not authenticated"
- )
+ raise ValueError(f"Connection '{connection_id}' not found or not authenticated")
if not connector.is_authenticated:
raise ValueError(f"Connection '{connection_id}' not authenticated")
@@ -334,7 +352,7 @@ class LangflowConnectorService:
# If file_infos provided, cache them in the connector for later use
# This allows get_file_content to use download URLs directly
- if file_infos and hasattr(connector, 'set_file_infos'):
+ if file_infos and hasattr(connector, "set_file_infos"):
connector.set_file_infos(file_infos)
logger.info(f"Cached {len(file_infos)} file infos with download URLs in connector")
@@ -357,8 +375,8 @@ class LangflowConnectorService:
# carefully selected IDs passed in.
if cfg is not None:
try:
- cfg.file_ids = file_ids # type: ignore
- cfg.folder_ids = None # type: ignore
+ cfg.file_ids = file_ids
+ cfg.folder_ids = None
# Expand file IDs — folders become their individual file contents
result = await connector.list_files()
@@ -383,8 +401,8 @@ class LangflowConnectorService:
# Fallback to original file_ids if expansion fails
expanded_file_ids = file_ids
finally:
- cfg.file_ids = original_file_ids # type: ignore
- cfg.folder_ids = original_folder_ids # type: ignore
+ cfg.file_ids = original_file_ids
+ cfg.folder_ids = original_folder_ids
processor = LangflowConnectorFileProcessor(
self,
@@ -401,9 +419,7 @@ class LangflowConnectorService:
original_filenames = {}
if file_infos:
original_filenames = {
- f["id"]: clean_connector_filename(
- f["name"], f.get("mimeType") or f.get("mimetype")
- )
+ f["id"]: clean_connector_filename(f["name"], f.get("mimeType") or f.get("mimetype"))
for f in file_infos
if "id" in f and "name" in f
}
@@ -414,6 +430,6 @@ class LangflowConnectorService:
return task_id
- async def _get_connector(self, connection_id: str) -> Optional[BaseConnector]:
+ async def _get_connector(self, connection_id: str) -> BaseConnector | None:
"""Get a connector by connection ID (alias for get_connector)"""
return await self.get_connector(connection_id)
diff --git a/src/models/processors.py b/src/models/processors.py
index 01df83a88..02771678a 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -303,6 +303,36 @@ class TaskProcessor:
opensearch_client, embedding_model, get_index_name(), dimensions
)
+ # Clear stale chunks from a prior indexing of this document. Chunks are
+ # stored under ids {file_hash}_{i}; if the new chunk count is lower than
+ # the prior one, trailing chunks (e.g. with an old filename after a
+ # SharePoint rename) would otherwise survive the per-chunk upsert.
+ # DLS-safe: enumerate then delete by primary _id (delete_by_query is
+ # silently filtered under DLS).
+ try:
+ from utils.opensearch_delete import (
+ collect_visible_document_ids,
+ delete_document_ids,
+ )
+
+ stale_chunk_ids = await collect_visible_document_ids(
+ opensearch_client,
+ index=get_index_name(),
+ query={"term": {"document_id": file_hash}},
+ )
+ await delete_document_ids(
+ opensearch_client,
+ index=get_index_name(),
+ document_ids=stale_chunk_ids,
+ refresh=True,
+ )
+ except Exception as e:
+ logger.warning(
+ "Failed to clear stale chunks before re-index; proceeding",
+ file_hash=file_hash,
+ error=str(e),
+ )
+
# Index each chunk as a separate document
for i, (chunk, vect) in enumerate(zip(slim_doc["chunks"], embeddings, strict=True)):
chunk_doc = {
diff --git a/src/utils/opensearch_delete.py b/src/utils/opensearch_delete.py
new file mode 100644
index 000000000..4e9db031f
--- /dev/null
+++ b/src/utils/opensearch_delete.py
@@ -0,0 +1,75 @@
+from collections.abc import Iterable
+
+from opensearchpy.exceptions import NotFoundError
+
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+async def collect_visible_document_ids(
+ opensearch_client,
+ *,
+ index: str,
+ query: dict,
+ source: bool | list[str] = False,
+ page_size: int = 1000,
+ scroll_ttl: str = "2m",
+) -> list[str]:
+ """Collect visible document IDs for a query using the caller's OpenSearch client."""
+ search_body = {"query": query, "size": page_size, "_source": source}
+ response = await opensearch_client.search(
+ index=index,
+ body=search_body,
+ scroll=scroll_ttl,
+ )
+ scroll_id = response.get("_scroll_id")
+ document_ids: list[str] = []
+
+ try:
+ while True:
+ hits = response.get("hits", {}).get("hits", [])
+ document_ids.extend(hit["_id"] for hit in hits if hit.get("_id"))
+ if not hits or len(hits) < page_size or not scroll_id:
+ break
+ response = await opensearch_client.scroll(scroll_id=scroll_id, scroll=scroll_ttl)
+ scroll_id = response.get("_scroll_id") or scroll_id
+ finally:
+ if scroll_id and hasattr(opensearch_client, "clear_scroll"):
+ try:
+ await opensearch_client.clear_scroll(scroll_id=scroll_id)
+ except Exception as e:
+ logger.debug("Failed to clear OpenSearch scroll context", error=str(e))
+
+ return document_ids
+
+
+async def delete_document_ids(
+ opensearch_client,
+ *,
+ index: str,
+ document_ids: Iterable[str],
+ refresh: bool = True,
+) -> int:
+ """Delete concrete OpenSearch document IDs through the caller's client.
+
+ delete_by_query is silently no-opped under DLS / certain security plugins
+ (returns deleted:N but leaves the docs in place). Single deletes keyed on
+ the primary _id are reliable, so enumerate visible IDs first and then issue
+ a delete per ID.
+ """
+ deleted_count = 0
+ for document_id in document_ids:
+ try:
+ result = await opensearch_client.delete(
+ index=index,
+ id=document_id,
+ refresh=refresh,
+ )
+ except NotFoundError:
+ continue
+
+ if result.get("result") == "deleted":
+ deleted_count += 1
+
+ return deleted_count
diff --git a/tests/unit/api/test_delete_chunks_by_document_ids.py b/tests/unit/api/test_delete_chunks_by_document_ids.py
index 005a1477d..5ae43fc49 100644
--- a/tests/unit/api/test_delete_chunks_by_document_ids.py
+++ b/tests/unit/api/test_delete_chunks_by_document_ids.py
@@ -2,8 +2,9 @@
Pins the contract of `src/api/documents.py::delete_chunks_by_document_ids`:
- empty input is a no-op (no OpenSearch call)
-- non-empty input issues a single delete_by_query with terms(document_id, ...)
-- returns the count reported by OpenSearch
+- non-empty input enumerates visible chunk _ids and deletes them one-by-one
+ by primary id (DLS-safe; `delete_by_query` is silently filtered under DLS)
+- returns the count of successful single deletes
"""
import sys
@@ -26,37 +27,61 @@ async def test_empty_ids_short_circuits_without_calling_opensearch():
deleted = await delete_chunks_by_document_ids([], opensearch_client, "test-index")
assert deleted == 0
+ opensearch_client.search.assert_not_awaited()
+ opensearch_client.delete.assert_not_awaited()
opensearch_client.delete_by_query.assert_not_awaited()
@pytest.mark.asyncio
-async def test_issues_single_delete_by_query_with_terms_filter():
+async def test_deletes_each_visible_chunk_id_by_primary_id():
+ """The helper must enumerate visible chunk _ids and issue a primary-id
+ delete for each. delete_by_query is forbidden because DLS / certain
+ security plugins silently no-op it (returns deleted:N but leaves docs)."""
from api.documents import delete_chunks_by_document_ids
opensearch_client = AsyncMock()
- opensearch_client.delete_by_query.return_value = {"deleted": 12}
+ # Visible chunk _ids for the requested document_ids.
+ chunk_ids = ["chunk-1", "chunk-2", "chunk-3"]
+ opensearch_client.search.return_value = {
+ "_scroll_id": None,
+ "hits": {"hits": [{"_id": cid} for cid in chunk_ids]},
+ }
+ opensearch_client.delete.return_value = {"result": "deleted"}
- ids = ["abc", "def", "ghi"]
- deleted = await delete_chunks_by_document_ids(ids, opensearch_client, "test-index")
+ deleted = await delete_chunks_by_document_ids(
+ ["doc-a", "doc-b"], opensearch_client, "test-index"
+ )
- assert deleted == 12
- opensearch_client.delete_by_query.assert_awaited_once()
- call = opensearch_client.delete_by_query.await_args
- assert call.kwargs["index"] == "test-index"
- assert call.kwargs["body"] == {"query": {"terms": {"document_id": ids}}}
- # conflicts="proceed" so a concurrent reindex doesn't abort the bulk delete.
- assert call.kwargs.get("conflicts") == "proceed"
+ assert deleted == len(chunk_ids)
+
+ # delete_by_query must NOT be used — silently filtered under DLS.
+ opensearch_client.delete_by_query.assert_not_awaited()
+
+ # Search query must target the document_id field with terms(...).
+ search_call = opensearch_client.search.await_args
+ assert search_call.kwargs["index"] == "test-index"
+ assert search_call.kwargs["body"]["query"] == {"terms": {"document_id": ["doc-a", "doc-b"]}}
+
+ # One primary-id delete per visible chunk, refresh=True so the delete is
+ # immediately visible to the re-index that typically follows.
+ delete_calls = opensearch_client.delete.await_args_list
+ assert len(delete_calls) == len(chunk_ids)
+ for call, cid in zip(delete_calls, chunk_ids, strict=True):
+ assert call.kwargs["index"] == "test-index"
+ assert call.kwargs["id"] == cid
+ assert call.kwargs.get("refresh") is True
@pytest.mark.asyncio
-async def test_returns_zero_when_response_missing_deleted_field():
- """Defensive: if OpenSearch returns an unexpected payload shape, we
- should not crash — we surface 0 deletions."""
+async def test_returns_zero_when_no_visible_chunks_match():
+ """When the search returns no hits, no deletes are issued and the count
+ is zero. This is the steady-state path (nothing to clean up)."""
from api.documents import delete_chunks_by_document_ids
opensearch_client = AsyncMock()
- opensearch_client.delete_by_query.return_value = {} # no "deleted" key
+ opensearch_client.search.return_value = {"_scroll_id": None, "hits": {"hits": []}}
deleted = await delete_chunks_by_document_ids(["abc"], opensearch_client, "test-index")
assert deleted == 0
+ opensearch_client.delete.assert_not_awaited()
diff --git a/tests/unit/test_processors_clear_stale_chunks.py b/tests/unit/test_processors_clear_stale_chunks.py
new file mode 100644
index 000000000..5c0246525
--- /dev/null
+++ b/tests/unit/test_processors_clear_stale_chunks.py
@@ -0,0 +1,219 @@
+"""Re-indexing in `process_document_standard` must clear prior chunks for the
+same `document_id` before writing the new ones.
+
+Per-chunk indexing uses ids `{file_hash}_{i}` with `opensearch_client.index`
+(upsert). Without a pre-delete, a re-index that produces fewer chunks than the
+prior pass leaves trailing chunks `{file_hash}_{N..M-1}` behind with the OLD
+metadata — most visibly, the old filename after a SharePoint rename. This test
+pins the invariant introduced to fix that.
+
+Pins: `src/models/processors.py` :: TaskProcessor.process_document_standard.
+"""
+
+import sys
+import tempfile
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+def _make_processor_with_mocks():
+ """Build a TaskProcessor wired to mocks for every external dependency
+ `process_document_standard` reaches. Returns (processor, opensearch_client)."""
+ from models.processors import TaskProcessor
+
+ opensearch_client = AsyncMock()
+ # exists() is checked at the top of process_document_standard; returning
+ # False forces the re-index path (the path where delete_by_query must fire).
+ opensearch_client.exists = AsyncMock(return_value=False)
+
+ session_manager = MagicMock()
+ session_manager.get_user_opensearch_client = MagicMock(return_value=opensearch_client)
+
+ document_service = MagicMock()
+ document_service.session_manager = session_manager
+
+ models_service = MagicMock()
+ models_service.get_litellm_model_name = AsyncMock(return_value="text-embedding-3-small")
+
+ docling_service = MagicMock() # unused for .txt path
+
+ processor = TaskProcessor(
+ document_service=document_service,
+ models_service=models_service,
+ docling_service=docling_service,
+ )
+ return processor, opensearch_client
+
+
+def _patch_embedding_pipeline(monkeypatch, chunk_count: int):
+ """Stub out the docling / embedding / index-mapping side of
+ process_document_standard so the test focuses on the OpenSearch write
+ ordering. `chunk_count` controls how many chunks the simulated text-file
+ parse produces.
+ """
+ from models import processors as processors_mod
+
+ fake_slim_doc = {
+ "id": "doc",
+ "filename": "ignored.txt",
+ "mimetype": "text/plain",
+ "chunks": [{"page": 1, "text": f"chunk-{i}"} for i in range(chunk_count)],
+ }
+ monkeypatch.setattr(processors_mod, "process_text_file", lambda _path: fake_slim_doc)
+
+ # Embedding model resolution path (config + fallback).
+ fake_config = MagicMock()
+ fake_config.knowledge.embedding_model = "text-embedding-3-small"
+ monkeypatch.setattr(processors_mod, "get_openrag_config", lambda: fake_config)
+ monkeypatch.setattr(processors_mod, "get_embedding_model", lambda: "text-embedding-3-small")
+ monkeypatch.setattr(processors_mod, "get_index_name", lambda: "test-index")
+
+ # Field-mapping helper writes nothing useful for this test.
+ async def _ensure_embedding_field_exists(_client, _model, _index, _dims):
+ return "embedding_field"
+
+ monkeypatch.setattr(
+ processors_mod, "ensure_embedding_field_exists", _ensure_embedding_field_exists
+ )
+
+ # chunk_texts_for_embeddings is imported lazily inside the function from
+ # services.document_service — patch it at its source.
+ from services import document_service as ds_mod
+
+ monkeypatch.setattr(
+ ds_mod,
+ "chunk_texts_for_embeddings",
+ lambda texts, max_tokens=8000: [list(texts)],
+ )
+
+ # patched_embedding_client.embeddings.create — return one embedding per text.
+ # `clients` is the singleton imported at module scope; replace it wholesale
+ # (the real one's `patched_embedding_client` is a read-only @property).
+ class _FakeEmbedResp:
+ def __init__(self, n):
+ self.data = [{"embedding": [0.1, 0.2, 0.3]} for _ in range(n)]
+
+ fake_embed_client = MagicMock()
+ fake_embed_client.embeddings.create = AsyncMock(
+ side_effect=lambda model, input: _FakeEmbedResp(len(input))
+ )
+ fake_clients = MagicMock()
+ fake_clients.patched_embedding_client = fake_embed_client
+ monkeypatch.setattr(processors_mod, "clients", fake_clients)
+
+
+async def test_stale_chunks_cleared_before_reindex(monkeypatch):
+ """Stale chunks must be cleared (via primary-id deletes) before any new
+ `index()` call so prior chunks (e.g. with an old filename after a rename)
+ cannot survive the per-chunk upsert.
+
+ DLS-safe pattern: enumerate visible chunk _ids via search, then issue a
+ `delete` per primary `_id`. `delete_by_query` is silently filtered under
+ DLS and must NOT be used.
+ """
+ processor, opensearch_client = _make_processor_with_mocks()
+ _patch_embedding_pipeline(monkeypatch, chunk_count=3)
+
+ stale_chunk_ids = ["abc123_0", "abc123_1", "abc123_2", "abc123_3", "abc123_4"]
+ op_order: list[tuple[str, dict]] = []
+
+ async def _search(**kw):
+ op_order.append(("search", kw))
+ return {"_scroll_id": None, "hits": {"hits": [{"_id": cid} for cid in stale_chunk_ids]}}
+
+ async def _delete(**kw):
+ op_order.append(("delete", kw))
+ return {"result": "deleted"}
+
+ async def _index(**kw):
+ op_order.append(("index", kw))
+
+ opensearch_client.search = AsyncMock(side_effect=_search)
+ opensearch_client.delete = AsyncMock(side_effect=_delete)
+ opensearch_client.index = AsyncMock(side_effect=_index)
+
+ with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:
+ tmp.write(b"hello world")
+ tmp_path = tmp.name
+
+ try:
+ await processor.process_document_standard(
+ file_path=tmp_path,
+ file_hash="abc123",
+ owner_user_id="alice",
+ original_filename="renamed.txt",
+ connector_type="sharepoint",
+ )
+ finally:
+ Path(tmp_path).unlink(missing_ok=True)
+
+ ops = [op for op, _ in op_order]
+ assert ops, "process_document_standard wrote nothing — fixture is broken"
+
+ # 1) The enumerate-via-search runs first.
+ assert ops[0] == "search", f"search must run before deletes. Saw: {ops}"
+ search_kwargs = op_order[0][1]
+ assert search_kwargs["body"]["query"] == {"term": {"document_id": "abc123"}}
+ assert search_kwargs["index"] == "test-index"
+
+ # 2) All deletes complete BEFORE any index() — they must precede re-indexing.
+ delete_indices = [i for i, op in enumerate(ops) if op == "delete"]
+ index_indices = [i for i, op in enumerate(ops) if op == "index"]
+ assert delete_indices, "no delete was issued; stale chunks would survive"
+ assert index_indices, "no chunks were indexed"
+ assert max(delete_indices) < min(index_indices), (
+ f"all deletes must complete before any index(). Saw order: {ops}"
+ )
+
+ # 3) One primary-id delete per visible stale chunk, refresh=True.
+ delete_calls = [kw for op, kw in op_order if op == "delete"]
+ assert len(delete_calls) == len(stale_chunk_ids)
+ for call, expected_id in zip(delete_calls, stale_chunk_ids, strict=True):
+ assert call["index"] == "test-index"
+ assert call["id"] == expected_id
+ assert call.get("refresh") is True
+
+ # 4) delete_by_query must NEVER be used (DLS would silently filter it).
+ if hasattr(opensearch_client, "delete_by_query"):
+ opensearch_client.delete_by_query.assert_not_called()
+
+
+async def test_delete_failure_does_not_abort_reindex(monkeypatch):
+ """A transient delete failure must be logged and swallowed — the per-chunk
+ upsert still runs so the sync isn't worse off than today's behavior."""
+ processor, opensearch_client = _make_processor_with_mocks()
+ _patch_embedding_pipeline(monkeypatch, chunk_count=2)
+
+ # Have the enumerate step itself blow up — that's the only "delete failure"
+ # surface the helper exposes, since per-id delete swallows NotFoundError.
+ opensearch_client.search = AsyncMock(side_effect=RuntimeError("os 503"))
+ opensearch_client.delete = AsyncMock()
+ index_calls: list[dict] = []
+ opensearch_client.index = AsyncMock(side_effect=lambda **kw: index_calls.append(kw))
+
+ with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as tmp:
+ tmp.write(b"hello world")
+ tmp_path = tmp.name
+
+ try:
+ result = await processor.process_document_standard(
+ file_path=tmp_path,
+ file_hash="abc123",
+ owner_user_id="alice",
+ original_filename="renamed.txt",
+ connector_type="sharepoint",
+ )
+ finally:
+ Path(tmp_path).unlink(missing_ok=True)
+
+ assert result["status"] == "indexed"
+ assert len(index_calls) == 2, "indexing must still happen if the pre-delete fails"