OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
"""Contract of connectors.chunk_cleanup — the single deletion path for a
connector file's indexed chunks (orphan reconcile, 404 cleanup, rename
cleanup all go through it).
- empty input is a no-op (no OpenSearch call)
- the query matches BOTH id layouts (document_id and connector_file_id)
- DLS-safe: visible chunk _ids are enumerated with the caller's client, then
deleted one-by-one by primary id with the trusted backend write client
(`delete_by_query` is silently filtered under DLS)
- owner/shared scoping and rename keep_filenames shape the query
"""
from unittest.mock import AsyncMock
import pytest
from connectors.chunk_cleanup import (
build_connector_file_chunks_query,
delete_connector_file_chunks,
)
@pytest.fixture
def write_client(monkeypatch):
import config.settings as cfg
client = AsyncMock()
client.delete = AsyncMock(return_value={"result": "deleted"})
monkeypatch.setattr(cfg.clients, "opensearch", client)
return client
def _id_terms(query) -> dict:
shoulds = query["bool"]["filter"][0]["bool"]["should"]
return {next(iter(c["terms"])): next(iter(c["terms"].values())) for c in shoulds}
def test_query_matches_both_id_fields():
query = build_connector_file_chunks_query(["file-1", "file-2"])
fields = _id_terms(query)
assert fields == {
"document_id": ["file-1", "file-2"],
"connector_file_id": ["file-1", "file-2"],
"connector_file_id.keyword": ["file-1", "file-2"],
}
assert query["bool"]["filter"][0]["bool"]["minimum_should_match"] == 1
def test_query_owner_scoping():
query = build_connector_file_chunks_query(["f"], owner_user_id="alice")
assert {"term": {"owner": "alice"}} in query["bool"]["filter"]
def test_query_shared_targets_ownerless_chunks():
query = build_connector_file_chunks_query(["f"], owner_user_id="alice", shared=True)
assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in query["bool"]["filter"]
assert {"term": {"owner": "alice"}} not in query["bool"]["filter"]
def test_query_include_shared_covers_owned_and_ownerless():
"""Sync scope: the file id pins the document down, so both layouts are ours.
A COS file ingested with the share-all toggle has no owner field at all; an
owner-only term matches none of its chunks and the cleanup silently no-ops.
"""
query = build_connector_file_chunks_query(["f"], owner_user_id="alice", include_shared=True)
assert {
"bool": {
"should": [
{"term": {"owner": "alice"}},
{"bool": {"must_not": {"exists": {"field": "owner"}}}},
],
"minimum_should_match": 1,
}
} in query["bool"]["filter"]
assert {"term": {"owner": "alice"}} not in query["bool"]["filter"]
def test_query_include_shared_ignored_without_owner():
query = build_connector_file_chunks_query(["f"], include_shared=True)
assert len(query["bool"]["filter"]) == 1
def test_query_shared_wins_over_include_shared():
"""shared=True is already ownerless-only; include_shared must not widen it."""
query = build_connector_file_chunks_query(
["f"], owner_user_id="alice", shared=True, include_shared=True
)
assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in query["bool"]["filter"]
assert not any("should" in str(f) and "owner" in str(f) for f in query["bool"]["filter"][1:])
def test_query_keep_filenames_excluded():
query = build_connector_file_chunks_query(["f"], keep_filenames=["new.pdf", "new_.pdf"])
assert query["bool"]["must_not"] == [{"terms": {"filename": ["new.pdf", "new_.pdf"]}}]
def test_query_unscoped_when_no_owner():
query = build_connector_file_chunks_query(["f"])
assert len(query["bool"]["filter"]) == 1
assert "must_not" not in query["bool"]
def test_query_connector_type_filter():
query = build_connector_file_chunks_query(["f"], connector_type="ibm_cos")
assert {"term": {"connector_type": "ibm_cos"}} in query["bool"]["filter"]
def test_query_connector_type_with_private_owner():
query = build_connector_file_chunks_query(
["f"], connector_type="google_drive", owner_user_id="alice"
)
filters = query["bool"]["filter"]
assert {"term": {"connector_type": "google_drive"}} in filters
assert {"term": {"owner": "alice"}} in filters
def test_query_connector_type_with_shared_scope():
query = build_connector_file_chunks_query(["f"], connector_type="ibm_cos", shared=True)
filters = query["bool"]["filter"]
assert {"term": {"connector_type": "ibm_cos"}} in filters
assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in filters
assert {"term": {"owner": "alice"}} not in filters
def test_query_no_connector_type_when_none():
query = build_connector_file_chunks_query(["f"])
for f in query["bool"]["filter"]:
if "term" in f:
assert "connector_type" not in f["term"]
@pytest.mark.asyncio
async def test_empty_ids_short_circuit_without_calling_opensearch(write_client):
opensearch_client = AsyncMock()
deleted = await delete_connector_file_chunks([], opensearch_client)
assert deleted == 0
opensearch_client.search.assert_not_awaited()
write_client.delete.assert_not_awaited()
@pytest.mark.asyncio
async def test_deletes_each_visible_chunk_id_by_primary_id(write_client, monkeypatch):
"""Visible chunk _ids are enumerated with the caller's client and deleted
one-by-one via the backend write client. delete_by_query is forbidden —
DLS / certain security plugins silently no-op it."""
monkeypatch.setattr("config.settings.get_index_name", lambda: "test-index")
opensearch_client = AsyncMock()
chunk_ids = ["chunk-1", "chunk-2", "chunk-3"]
opensearch_client.search.return_value = {
"_scroll_id": None,
"hits": {"hits": [{"_id": cid} for cid in chunk_ids]},
}
deleted = await delete_connector_file_chunks(
["doc-a", "doc-b"], opensearch_client, refresh=True
)
assert deleted == len(chunk_ids)
opensearch_client.delete_by_query.assert_not_awaited()
write_client.delete_by_query.assert_not_awaited()
search_call = opensearch_client.search.await_args
assert search_call.kwargs["index"] == "test-index"
fields = _id_terms(search_call.kwargs["body"]["query"])
assert fields["document_id"] == ["doc-a", "doc-b"]
assert fields["connector_file_id"] == ["doc-a", "doc-b"]
assert fields["connector_file_id.keyword"] == ["doc-a", "doc-b"]
opensearch_client.delete.assert_not_awaited()
delete_calls = write_client.delete.await_args_list
assert [call.kwargs["id"] for call in delete_calls] == chunk_ids
for call in delete_calls:
assert call.kwargs["index"] == "test-index"
assert call.kwargs.get("refresh") is True
@pytest.mark.asyncio
async def test_returns_zero_when_no_visible_chunks_match(write_client):
opensearch_client = AsyncMock()
opensearch_client.search.return_value = {"_scroll_id": None, "hits": {"hits": []}}
deleted = await delete_connector_file_chunks(["abc"], opensearch_client)
assert deleted == 0
write_client.delete.assert_not_awaited()
@pytest.mark.asyncio
async def test_raises_when_write_client_unavailable(monkeypatch):
import config.settings as cfg
monkeypatch.setattr(cfg.clients, "opensearch", None)
with pytest.raises(RuntimeError, match="write client"):
await delete_connector_file_chunks(["abc"], AsyncMock())