Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: speed up bulk document delete with batched _bulk requests (#2594)

VisveshJe9c85d3ff05a
diff --git a/src/api/documents.py b/src/api/documents.py
index f4901dace..7c8933c3f 100644
--- a/src/api/documents.py
+++ b/src/api/documents.py
@@ -33,7 +33,10 @@ async def delete_documents_by_filename_core(
):
"""Shared delete-by-filename logic for v1 and non-v1 endpoints."""
from config.settings import get_index_name
- from utils.opensearch_delete import collect_visible_document_ids, delete_document_ids
+ from utils.opensearch_delete import (
+ bulk_delete_document_ids,
+ collect_visible_document_ids,
+ )
from utils.opensearch_queries import (
build_anonymous_filename_query,
build_filename_query,
@@ -140,7 +143,7 @@ async def delete_documents_by_filename_core(
if write_client is None:
raise RuntimeError("Backend OpenSearch client is unavailable")
- deleted_count = await delete_document_ids(
+ deleted_count = await bulk_delete_document_ids(
write_client,
index=index_name,
document_ids=document_ids,
@@ -185,7 +188,8 @@ async def delete_documents_by_filename_core(
return (
{
"success": False,
- "deleted_chunks": 0,
+ # Chunks already confirmed deleted before a partial bulk failure.
+ "deleted_chunks": getattr(e, "deleted_count", 0),
"filename": normalized_filename,
"message": None,
"error": (
diff --git a/src/utils/opensearch_delete.py b/src/utils/opensearch_delete.py
index 84ee34bce..3ce67742e 100644
--- a/src/utils/opensearch_delete.py
+++ b/src/utils/opensearch_delete.py
@@ -78,3 +78,82 @@ async def delete_document_ids(
invalidate_corpus_caches()
return deleted_count
+
+
+class BulkDeleteError(RuntimeError):
+ """A bulk delete failed part-way; ``deleted_count`` holds confirmed deletions.
+
+ Only deletions acknowledged by a successful ``_bulk`` response are counted.
+ If a request itself failed, any deletions it may have applied are unknown
+ and are not included.
+ """
+
+ def __init__(self, message: str, deleted_count: int = 0):
+ super().__init__(message)
+ self.deleted_count = deleted_count
+
+
+async def bulk_delete_document_ids(
+ opensearch_client,
+ *,
+ index: str,
+ document_ids: Iterable[str],
+ refresh: bool = True,
+ batch_size: int = 500,
+) -> int:
+ """Delete concrete OpenSearch document IDs using batched ``_bulk`` requests.
+
+ Same ID-keyed delete semantics as :func:`delete_document_ids` (so it stays
+ DLS-safe), but one request per ``batch_size`` IDs instead of one per ID, and
+ a single index refresh at the end instead of one per delete. Intended for
+ user-facing deletes of many chunks, where per-ID round trips and per-delete
+ refreshes dominate latency.
+
+ When ``refresh`` is true the index is refreshed once after any confirmed
+ deletions, including before a failure is propagated. A per-request
+ ``refresh=true`` would only cover the shards that request touched, so it is
+ not used. A failed refresh is logged and never masks the delete result or
+ the original error; corpus caches are invalidated regardless.
+
+ Missing documents (404) are ignored. Any other per-item failure is raised as
+ a :class:`BulkDeleteError` after all batches have been attempted. A failed
+ request stops further batches and is also raised as a ``BulkDeleteError``
+ (chained to the original exception). Either way, ``deleted_count`` on the
+ error carries the deletions confirmed so far.
+
+ Raises ``ValueError`` if ``batch_size`` is not positive.
+ """
+ if batch_size <= 0:
+ raise ValueError("batch_size must be > 0")
+ ids = list(document_ids)
+ deleted_count = 0
+ first_error: str | None = None
+ try:
+ for start in range(0, len(ids), batch_size):
+ batch = ids[start : start + batch_size]
+ try:
+ response = await opensearch_client.bulk(
+ body=[{"delete": {"_index": index, "_id": doc_id}} for doc_id in batch],
+ refresh=False,
+ )
+ except Exception as e:
+ raise BulkDeleteError(str(e), deleted_count) from e
+ for item in response.get("items", []):
+ action = item.get("delete", {})
+ if action.get("result") == "deleted":
+ deleted_count += 1
+ elif action.get("status") != 404 and first_error is None:
+ first_error = str(action.get("error") or action.get("status"))
+ finally:
+ if deleted_count:
+ try:
+ if refresh:
+ await opensearch_client.indices.refresh(index=index)
+ except Exception as e:
+ logger.warning("Failed to refresh index after bulk delete", error=str(e))
+ finally:
+ invalidate_corpus_caches()
+
+ if first_error is not None:
+ raise BulkDeleteError(f"Bulk delete failed for some chunks: {first_error}", deleted_count)
+ return deleted_count
diff --git a/tests/unit/test_bulk_delete_document_ids.py b/tests/unit/test_bulk_delete_document_ids.py
new file mode 100644
index 000000000..047cda099
--- /dev/null
+++ b/tests/unit/test_bulk_delete_document_ids.py
@@ -0,0 +1,196 @@
+import pytest
+
+from services import search_cache
+from utils.opensearch_delete import BulkDeleteError, bulk_delete_document_ids
+
+
+class _Indices:
+ def __init__(self, error=None):
+ self.error = error
+ self.refresh_calls = []
+
+ async def refresh(self, *, index):
+ self.refresh_calls.append(index)
+ if self.error:
+ raise self.error
+
+
+class _BulkClient:
+ """Fake client; ``outcomes`` maps document id -> (result, status[, error])."""
+
+ def __init__(self, outcomes=None, refresh_error=None):
+ self.outcomes = outcomes or {}
+ self.calls = []
+ self.indices = _Indices(refresh_error)
+
+ async def bulk(self, *, body, refresh=False):
+ self.calls.append({"body": body, "refresh": refresh})
+ items = []
+ for action in body:
+ doc_id = action["delete"]["_id"]
+ result, status, *error = self.outcomes.get(doc_id, ("deleted", 200))
+ entry = {"_id": doc_id, "result": result, "status": status}
+ if error:
+ entry["error"] = error[0]
+ items.append({"delete": entry})
+ return {"errors": any(i["delete"]["status"] >= 400 for i in items), "items": items}
+
+
+def _prime_caches():
+ search_cache.SUGGEST_CACHE.set("g", {"files": ["old.pdf"]})
+
+
+async def test_batches_ids_and_refreshes_the_index_once_after_all_batches():
+ client = _BulkClient()
+
+ deleted = await bulk_delete_document_ids(
+ client, index="documents", document_ids=[f"c{i}" for i in range(5)], batch_size=2
+ )
+
+ assert deleted == 5
+ assert [len(c["body"]) for c in client.calls] == [2, 2, 1]
+ assert [c["refresh"] for c in client.calls] == [False, False, False]
+ assert client.calls[0]["body"][0] == {"delete": {"_index": "documents", "_id": "c0"}}
+ assert client.indices.refresh_calls == ["documents"]
+
+
+async def test_refresh_false_never_refreshes():
+ client = _BulkClient()
+
+ await bulk_delete_document_ids(
+ client, index="documents", document_ids=["a", "b", "c"], batch_size=2, refresh=False
+ )
+
+ assert [c["refresh"] for c in client.calls] == [False, False]
+ assert client.indices.refresh_calls == []
+
+
+async def test_nothing_deleted_does_not_refresh():
+ client = _BulkClient({"a": ("not_found", 404)})
+
+ await bulk_delete_document_ids(client, index="documents", document_ids=["a"])
+
+ assert client.indices.refresh_calls == []
+
+
+async def test_refresh_failure_is_swallowed_and_caches_are_still_dropped():
+ _prime_caches()
+ client = _BulkClient(refresh_error=ConnectionError("refresh down"))
+
+ deleted = await bulk_delete_document_ids(client, index="documents", document_ids=["a", "b"])
+
+ assert deleted == 2
+ assert client.indices.refresh_calls == ["documents"]
+ assert search_cache.SUGGEST_CACHE.get("g") is None
+
+
+async def test_empty_ids_make_no_request():
+ client = _BulkClient()
+
+ assert await bulk_delete_document_ids(client, index="documents", document_ids=[]) == 0
+ assert client.calls == []
+
+
+async def test_missing_documents_are_ignored():
+ client = _BulkClient({"a": ("not_found", 404)})
+
+ deleted = await bulk_delete_document_ids(client, index="documents", document_ids=["a", "b"])
+
+ assert deleted == 1
+
+
+async def test_item_failure_raises_after_all_batches_and_still_drops_caches():
+ _prime_caches()
+ client = _BulkClient({"a": ("error", 500, {"type": "boom"})})
+
+ with pytest.raises(RuntimeError, match="boom"):
+ await bulk_delete_document_ids(
+ client, index="documents", document_ids=["a", "b", "c"], batch_size=1
+ )
+
+ assert len(client.calls) == 3
+ assert client.indices.refresh_calls == ["documents"]
+ assert search_cache.SUGGEST_CACHE.get("g") is None
+
+
+async def test_nothing_deleted_keeps_caches():
+ _prime_caches()
+ client = _BulkClient({"a": ("not_found", 404)})
+
+ await bulk_delete_document_ids(client, index="documents", document_ids=["a"])
+
+ assert search_cache.SUGGEST_CACHE.get("g") is not None
+
+
+async def test_item_failure_reports_confirmed_deletions():
+ client = _BulkClient({"b": ("error", 500, {"type": "boom"})})
+
+ with pytest.raises(BulkDeleteError) as exc_info:
+ await bulk_delete_document_ids(
+ client, index="documents", document_ids=["a", "b", "c"], batch_size=2
+ )
+
+ assert exc_info.value.deleted_count == 2
+
+
+class _FailingSecondBatchClient(_BulkClient):
+ async def bulk(self, *, body, refresh=False):
+ if self.calls:
+ self.calls.append({"body": body, "refresh": refresh})
+ raise ConnectionError("opensearch down")
+ return await super().bulk(body=body, refresh=refresh)
+
+
+async def test_request_failure_reports_only_confirmed_deletions_and_stops():
+ _prime_caches()
+ client = _FailingSecondBatchClient()
+
+ with pytest.raises(BulkDeleteError, match="opensearch down") as exc_info:
+ await bulk_delete_document_ids(
+ client, index="documents", document_ids=["a", "b", "c", "d", "e"], batch_size=2
+ )
+
+ assert exc_info.value.deleted_count == 2
+ assert isinstance(exc_info.value.__cause__, ConnectionError)
+ assert len(client.calls) == 2
+ # Batch 1's deletions are made visible before the failure propagates.
+ assert client.indices.refresh_calls == ["documents"]
+ assert search_cache.SUGGEST_CACHE.get("g") is None
+
+
+async def test_refresh_failure_does_not_replace_the_original_bulk_error():
+ _prime_caches()
+ client = _FailingSecondBatchClient(refresh_error=ConnectionError("refresh down"))
+
+ with pytest.raises(BulkDeleteError, match="opensearch down") as exc_info:
+ await bulk_delete_document_ids(
+ client, index="documents", document_ids=["a", "b", "c"], batch_size=2
+ )
+
+ assert exc_info.value.deleted_count == 2
+ assert search_cache.SUGGEST_CACHE.get("g") is None
+
+
[email protected]("batch_size", [0, -1])
+async def test_non_positive_batch_size_is_rejected_before_any_request(batch_size):
+ client = _BulkClient()
+
+ with pytest.raises(ValueError, match="batch_size must be > 0"):
+ await bulk_delete_document_ids(
+ client, index="documents", document_ids=["a"], batch_size=batch_size
+ )
+
+ assert client.calls == []
+ assert client.indices.refresh_calls == []
diff --git a/tests/unit/test_document_delete_by_id.py b/tests/unit/test_document_delete_by_id.py
index c8844434d..b03e70538 100644
--- a/tests/unit/test_document_delete_by_id.py
+++ b/tests/unit/test_document_delete_by_id.py
@@ -3,12 +3,22 @@ import pytest
from api.documents import delete_documents_by_filename_core
+class _Indices:
+ def __init__(self):
+ self.refresh_calls = []
+
+ async def refresh(self, *, index):
+ self.refresh_calls.append(index)
+
+
class FakeOpenSearchClient:
def __init__(self, owned_hits=None, visible_hits=None):
self.owned_hits = owned_hits or []
self.visible_hits = visible_hits or []
self.search_calls = []
self.delete_calls = []
+ self.bulk_calls = []
+ self.indices = _Indices()
async def search(self, *, index, body, scroll=None):
self.search_calls.append({"index": index, "body": body, "scroll": scroll})
@@ -23,6 +33,24 @@ class FakeOpenSearchClient:
self.delete_calls.append({"index": index, "id": id, "refresh": refresh})
return {"result": "deleted"}
+ async def bulk(self, *, body, refresh=False):
+ self.bulk_calls.append({"body": body, "refresh": refresh})
+ for action in body:
+ self.delete_calls.append(
+ {
+ "index": action["delete"]["_index"],
+ "id": action["delete"]["_id"],
+ "refresh": refresh,
+ }
+ )
+ return {
+ "errors": False,
+ "items": [
+ {"delete": {"_id": a["delete"]["_id"], "result": "deleted", "status": 200}}
+ for a in body
+ ],
+ }
+
class FakeSessionManager:
def __init__(self, opensearch_client):
@@ -67,9 +95,10 @@ async def test_delete_documents_by_filename_deletes_owned_ids_with_backend_clien
}
assert opensearch_client.delete_calls == []
assert backend_opensearch_client.delete_calls == [
- {"index": "documents", "id": "chunk-1", "refresh": True},
- {"index": "documents", "id": "chunk-2", "refresh": True},
+ {"index": "documents", "id": "chunk-1", "refresh": False},
+ {"index": "documents", "id": "chunk-2", "refresh": False},
]
+ assert backend_opensearch_client.indices.refresh_calls == ["documents"]
@pytest.mark.asyncio
@@ -132,9 +161,10 @@ async def test_delete_documents_by_filename_deletes_ownerless_with_anonymous_per
}
}
assert backend_opensearch_client.delete_calls == [
- {"index": "documents", "id": "shared-chunk-1", "refresh": True},
- {"index": "documents", "id": "shared-chunk-2", "refresh": True},
+ {"index": "documents", "id": "shared-chunk-1", "refresh": False},
+ {"index": "documents", "id": "shared-chunk-2", "refresh": False},
]
+ assert backend_opensearch_client.indices.refresh_calls == ["documents"]
@pytest.mark.asyncio
@@ -240,3 +270,38 @@ async def test_delete_documents_by_filename_returns_404_when_missing(monkeypatch
assert payload["success"] is False
assert payload["deleted_chunks"] == 0
assert opensearch_client.delete_calls == []
+
+
+async def test_delete_documents_by_filename_reports_confirmed_chunks_on_partial_failure(
+ monkeypatch,
+):
+ monkeypatch.setattr("config.settings.get_index_name", lambda: "documents")
+ opensearch_client = FakeOpenSearchClient(
+ owned_hits=[{"_id": "chunk-1", "_source": {"owner": "user-1"}}]
+ )
+ backend_opensearch_client = FakeOpenSearchClient()
+
+ async def _partial_bulk(*, body, refresh=False):
+ return {
+ "errors": True,
+ "items": [
+ {"delete": {"_id": "chunk-1", "result": "deleted", "status": 200}},
+ {"delete": {"_id": "chunk-2", "status": 500, "error": {"type": "boom"}}},
+ ],
+ }
+
+ backend_opensearch_client.bulk = _partial_bulk
+ monkeypatch.setattr("config.settings.clients.opensearch", backend_opensearch_client)
+
+ payload, status_code = await delete_documents_by_filename_core(
+ filename="report.pdf",
+ session_manager=FakeSessionManager(opensearch_client),
+ user_id="user-1",
+ jwt_token="jwt-token",
+ )
+
+ assert status_code == 500
+ assert payload["success"] is False
+ assert payload["deleted_chunks"] == 1
+ assert "boom" not in payload["error"]