OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: speed up bulk document delete with batched _bulk requests (#2594)
VisveshJe9c85d3ff05a
diff --git a/src/api/documents.py b/src/api/documents.pyindex f4901dace..7c8933c3f 100644--- a/src/api/documents.py+++ b/src/api/documents.py@@ -33,7 +33,10 @@ async def delete_documents_by_filename_core():"""Shared delete-by-filename logic for v1 and non-v1 endpoints."""from config.settings import get_index_name- from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids+ from utils.opensearch_delete import (+ bulk_delete_document_ids,+ collect_visible_document_ids,+ )from utils.opensearch_queries import (build_anonymous_filename_query,build_filename_query,@@ -140,7 +143,7 @@ async def delete_documents_by_filename_core(if write_client is None:raise RuntimeError("Backend OpenSearch client is unavailable")- deleted_count = await delete_document_ids(+ deleted_count = await bulk_delete_document_ids(write_client,index=index_name,document_ids=document_ids,@@ -185,7 +188,8 @@ async def delete_documents_by_filename_core(return ({"success": False,- "deleted_chunks": 0,+ # Chunks already confirmed deleted before a partial bulk failure.+ "deleted_chunks": getattr(e, "deleted_count", 0),"filename": normalized_filename,"message": None,"error": (diff --git a/src/utils/opensearch_delete.py b/src/utils/opensearch_delete.pyindex 84ee34bce..3ce67742e 100644--- a/src/utils/opensearch_delete.py+++ b/src/utils/opensearch_delete.py@@ -78,3 +78,82 @@ async def delete_document_ids(invalidate_corpus_caches()return deleted_count+++class BulkDeleteError(RuntimeError):+ """A bulk delete failed part-way; ``deleted_count`` holds confirmed deletions.++ Only deletions acknowledged by a successful ``_bulk`` response are counted.+ If a request itself failed, any deletions it may have applied are unknown+ and are not included.+ """++ def __init__(self, message: str, deleted_count: int = 0):+ super().__init__(message)+ self.deleted_count = deleted_count+++async def bulk_delete_document_ids(+ opensearch_client,+ *,+ index: str,+ document_ids: Iterable[str],+ refresh: bool = True,+ batch_size: int = 500,+) -> int:+ """Delete concrete OpenSearch document IDs using batched ``_bulk`` requests.++ Same ID-keyed delete semantics as :func:`delete_document_ids` (so it stays+ DLS-safe), but one request per ``batch_size`` IDs instead of one per ID, and+ a single index refresh at the end instead of one per delete. Intended for+ user-facing deletes of many chunks, where per-ID round trips and per-delete+ refreshes dominate latency.++ When ``refresh`` is true the index is refreshed once after any confirmed+ deletions, including before a failure is propagated. A per-request+ ``refresh=true`` would only cover the shards that request touched, so it is+ not used. A failed refresh is logged and never masks the delete result or+ the original error; corpus caches are invalidated regardless.++ Missing documents (404) are ignored. Any other per-item failure is raised as+ a :class:`BulkDeleteError` after all batches have been attempted. A failed+ request stops further batches and is also raised as a ``BulkDeleteError``+ (chained to the original exception). Either way, ``deleted_count`` on the+ error carries the deletions confirmed so far.++ Raises ``ValueError`` if ``batch_size`` is not positive.+ """+ if batch_size <= 0:+ raise ValueError("batch_size must be > 0")+ ids = list(document_ids)+ deleted_count = 0+ first_error: str | None = None+ try:+ for start in range(0, len(ids), batch_size):+ batch = ids[start : start + batch_size]+ try:+ response = await opensearch_client.bulk(+ body=[{"delete": {"_index": index, "_id": doc_id}} for doc_id in batch],+ refresh=False,+ )+ except Exception as e:+ raise BulkDeleteError(str(e), deleted_count) from e+ for item in response.get("items", []):+ action = item.get("delete", {})+ if action.get("result") == "deleted":+ deleted_count += 1+ elif action.get("status") != 404 and first_error is None:+ first_error = str(action.get("error") or action.get("status"))+ finally:+ if deleted_count:+ try:+ if refresh:+ await opensearch_client.indices.refresh(index=index)+ except Exception as e:+ logger.warning("Failed to refresh index after bulk delete", error=str(e))+ finally:+ invalidate_corpus_caches()++ if first_error is not None:+ raise BulkDeleteError(f"Bulk delete failed for some chunks: {first_error}", deleted_count)+ return deleted_countdiff --git a/tests/unit/test_bulk_delete_document_ids.py b/tests/unit/test_bulk_delete_document_ids.pynew file mode 100644index 000000000..047cda099--- /dev/null+++ b/tests/unit/test_bulk_delete_document_ids.py@@ -0,0 +1,196 @@+import pytest++from services import search_cache+from utils.opensearch_delete import BulkDeleteError, bulk_delete_document_ids+++class _Indices:+ def __init__(self, error=None):+ self.error = error+ self.refresh_calls = []++ async def refresh(self, *, index):+ self.refresh_calls.append(index)+ if self.error:+ raise self.error+++class _BulkClient:+ """Fake client; ``outcomes`` maps document id -> (result, status[, error])."""++ def __init__(self, outcomes=None, refresh_error=None):+ self.outcomes = outcomes or {}+ self.calls = []+ self.indices = _Indices(refresh_error)++ async def bulk(self, *, body, refresh=False):+ self.calls.append({"body": body, "refresh": refresh})+ items = []+ for action in body:+ doc_id = action["delete"]["_id"]+ result, status, *error = self.outcomes.get(doc_id, ("deleted", 200))+ entry = {"_id": doc_id, "result": result, "status": status}+ if error:+ entry["error"] = error[0]+ items.append({"delete": entry})+ return {"errors": any(i["delete"]["status"] >= 400 for i in items), "items": items}+++def _prime_caches():+ search_cache.SUGGEST_CACHE.set("g", {"files": ["old.pdf"]})+++async def test_batches_ids_and_refreshes_the_index_once_after_all_batches():+ client = _BulkClient()++ deleted = await bulk_delete_document_ids(+ client, index="documents", document_ids=[f"c{i}" for i in range(5)], batch_size=2+ )++ assert deleted == 5+ assert [len(c["body"]) for c in client.calls] == [2, 2, 1]+ assert [c["refresh"] for c in client.calls] == [False, False, False]+ assert client.calls[0]["body"][0] == {"delete": {"_index": "documents", "_id": "c0"}}+ assert client.indices.refresh_calls == ["documents"]+++async def test_refresh_false_never_refreshes():+ client = _BulkClient()++ await bulk_delete_document_ids(+ client, index="documents", document_ids=["a", "b", "c"], batch_size=2, refresh=False+ )++ assert [c["refresh"] for c in client.calls] == [False, False]+ assert client.indices.refresh_calls == []+++async def test_nothing_deleted_does_not_refresh():+ client = _BulkClient({"a": ("not_found", 404)})++ await bulk_delete_document_ids(client, index="documents", document_ids=["a"])++ assert client.indices.refresh_calls == []+++async def test_refresh_failure_is_swallowed_and_caches_are_still_dropped():+ _prime_caches()+ client = _BulkClient(refresh_error=ConnectionError("refresh down"))++ deleted = await bulk_delete_document_ids(client, index="documents", document_ids=["a", "b"])++ assert deleted == 2+ assert client.indices.refresh_calls == ["documents"]+ assert search_cache.SUGGEST_CACHE.get("g") is None+++async def test_empty_ids_make_no_request():+ client = _BulkClient()++ assert await bulk_delete_document_ids(client, index="documents", document_ids=[]) == 0+ assert client.calls == []+++async def test_missing_documents_are_ignored():+ client = _BulkClient({"a": ("not_found", 404)})++ deleted = await bulk_delete_document_ids(client, index="documents", document_ids=["a", "b"])++ assert deleted == 1+++async def test_item_failure_raises_after_all_batches_and_still_drops_caches():+ _prime_caches()+ client = _BulkClient({"a": ("error", 500, {"type": "boom"})})++ with pytest.raises(RuntimeError, match="boom"):+ await bulk_delete_document_ids(+ client, index="documents", document_ids=["a", "b", "c"], batch_size=1+ )++ assert len(client.calls) == 3+ assert client.indices.refresh_calls == ["documents"]+ assert search_cache.SUGGEST_CACHE.get("g") is None+++async def test_nothing_deleted_keeps_caches():+ _prime_caches()+ client = _BulkClient({"a": ("not_found", 404)})++ await bulk_delete_document_ids(client, index="documents", document_ids=["a"])++ assert search_cache.SUGGEST_CACHE.get("g") is not None+++async def test_item_failure_reports_confirmed_deletions():+ client = _BulkClient({"b": ("error", 500, {"type": "boom"})})++ with pytest.raises(BulkDeleteError) as exc_info:+ await bulk_delete_document_ids(+ client, index="documents", document_ids=["a", "b", "c"], batch_size=2+ )++ assert exc_info.value.deleted_count == 2+++class _FailingSecondBatchClient(_BulkClient):+ async def bulk(self, *, body, refresh=False):+ if self.calls:+ self.calls.append({"body": body, "refresh": refresh})+ raise ConnectionError("opensearch down")+ return await super().bulk(body=body, refresh=refresh)+++async def test_request_failure_reports_only_confirmed_deletions_and_stops():+ _prime_caches()+ client = _FailingSecondBatchClient()++ with pytest.raises(BulkDeleteError, match="opensearch down") as exc_info:+ await bulk_delete_document_ids(+ client, index="documents", document_ids=["a", "b", "c", "d", "e"], batch_size=2+ )++ assert exc_info.value.deleted_count == 2+ assert isinstance(exc_info.value.__cause__, ConnectionError)+ assert len(client.calls) == 2+ # Batch 1's deletions are made visible before the failure propagates.+ assert client.indices.refresh_calls == ["documents"]+ assert search_cache.SUGGEST_CACHE.get("g") is None+++async def test_refresh_failure_does_not_replace_the_original_bulk_error():+ _prime_caches()+ client = _FailingSecondBatchClient(refresh_error=ConnectionError("refresh down"))++ with pytest.raises(BulkDeleteError, match="opensearch down") as exc_info:+ await bulk_delete_document_ids(+ client, index="documents", document_ids=["a", "b", "c"], batch_size=2+ )++ assert exc_info.value.deleted_count == 2+ assert search_cache.SUGGEST_CACHE.get("g") is None++[email protected]("batch_size", [0, -1])+async def test_non_positive_batch_size_is_rejected_before_any_request(batch_size):+ client = _BulkClient()++ with pytest.raises(ValueError, match="batch_size must be > 0"):+ await bulk_delete_document_ids(+ client, index="documents", document_ids=["a"], batch_size=batch_size+ )++ assert client.calls == []+ assert client.indices.refresh_calls == []diff --git a/tests/unit/test_document_delete_by_id.py b/tests/unit/test_document_delete_by_id.pyindex c8844434d..b03e70538 100644--- a/tests/unit/test_document_delete_by_id.py+++ b/tests/unit/test_document_delete_by_id.py@@ -3,12 +3,22 @@ import pytestfrom api.documents import delete_documents_by_filename_core+class _Indices:+ def __init__(self):+ self.refresh_calls = []++ async def refresh(self, *, index):+ self.refresh_calls.append(index)++class FakeOpenSearchClient:def __init__(self, owned_hits=None, visible_hits=None):self.owned_hits = owned_hits or []self.visible_hits = visible_hits or []self.search_calls = []self.delete_calls = []+ self.bulk_calls = []+ self.indices = _Indices()async def search(self, *, index, body, scroll=None):self.search_calls.append({"index": index, "body": body, "scroll": scroll})@@ -23,6 +33,24 @@ class FakeOpenSearchClient:self.delete_calls.append({"index": index, "id": id, "refresh": refresh})return {"result": "deleted"}+ async def bulk(self, *, body, refresh=False):+ self.bulk_calls.append({"body": body, "refresh": refresh})+ for action in body:+ self.delete_calls.append(+ {+ "index": action["delete"]["_index"],+ "id": action["delete"]["_id"],+ "refresh": refresh,+ }+ )+ return {+ "errors": False,+ "items": [+ {"delete": {"_id": a["delete"]["_id"], "result": "deleted", "status": 200}}+ for a in body+ ],+ }+class FakeSessionManager:def __init__(self, opensearch_client):@@ -67,9 +95,10 @@ async def test_delete_documents_by_filename_deletes_owned_ids_with_backend_clien}assert opensearch_client.delete_calls == []assert backend_opensearch_client.delete_calls == [- {"index": "documents", "id": "chunk-1", "refresh": True},- {"index": "documents", "id": "chunk-2", "refresh": True},+ {"index": "documents", "id": "chunk-1", "refresh": False},+ {"index": "documents", "id": "chunk-2", "refresh": False},]+ assert backend_opensearch_client.indices.refresh_calls == ["documents"]@pytest.mark.asyncio@@ -132,9 +161,10 @@ async def test_delete_documents_by_filename_deletes_ownerless_with_anonymous_per}}assert backend_opensearch_client.delete_calls == [- {"index": "documents", "id": "shared-chunk-1", "refresh": True},- {"index": "documents", "id": "shared-chunk-2", "refresh": True},+ {"index": "documents", "id": "shared-chunk-1", "refresh": False},+ {"index": "documents", "id": "shared-chunk-2", "refresh": False},]+ assert backend_opensearch_client.indices.refresh_calls == ["documents"]@pytest.mark.asyncio@@ -240,3 +270,38 @@ async def test_delete_documents_by_filename_returns_404_when_missing(monkeypatchassert payload["success"] is Falseassert payload["deleted_chunks"] == 0assert opensearch_client.delete_calls == []+++async def test_delete_documents_by_filename_reports_confirmed_chunks_on_partial_failure(+ monkeypatch,+):+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents")+ opensearch_client = FakeOpenSearchClient(+ owned_hits=[{"_id": "chunk-1", "_source": {"owner": "user-1"}}]+ )+ backend_opensearch_client = FakeOpenSearchClient()++ async def _partial_bulk(*, body, refresh=False):+ return {+ "errors": True,+ "items": [+ {"delete": {"_id": "chunk-1", "result": "deleted", "status": 200}},+ {"delete": {"_id": "chunk-2", "status": 500, "error": {"type": "boom"}}},+ ],+ }++ backend_opensearch_client.bulk = _partial_bulk+ monkeypatch.setattr("config.settings.clients.opensearch", backend_opensearch_client)++ payload, status_code = await delete_documents_by_filename_core(+ filename="report.pdf",+ session_manager=FakeSessionManager(opensearch_client),+ user_id="user-1",+ jwt_token="jwt-token",+ )++ assert status_code == 500+ assert payload["success"] is False+ assert payload["deleted_chunks"] == 1+ assert "boom" not in payload["error"]