Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(search): honor the cached document-deleted flag so deleted docs stay pruned (#19070)

MarMar Labs95b51e89ad8f
diff --git a/rag/nlp/search.py b/rag/nlp/search.py
index 445b598880..51d5b28f79 100644
--- a/rag/nlp/search.py
+++ b/rag/nlp/search.py
@@ -91,7 +91,7 @@ class Dealer:
# Fast path: serve every doc_id from the short-lived cache if it is fresh.
with self._doc_exists_lock:
cached = {d: v for d, v in self._doc_exists_cache.items() if now - v[0] < self._DOC_EXISTS_TTL}
- hit = {d for d in unique_doc_ids if d in cached}
+ hit = {d for d in unique_doc_ids if d in cached and cached[d][1]}
miss = [d for d in unique_doc_ids if d not in cached]
if not miss:
diff --git a/test/unit_test/rag/test_doc_exists_cache.py b/test/unit_test/rag/test_doc_exists_cache.py
new file mode 100644
index 0000000000..1c3a2640da
--- /dev/null
+++ b/test/unit_test/rag/test_doc_exists_cache.py
@@ -0,0 +1,128 @@
+#
+# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Unit tests for the doc-existence cache behind Dealer._prune_deleted_chunks."""
+
+import sys
+import threading
+import types
+from collections import OrderedDict
+
+import pytest
+
+# Stub the heavy / circular-importing dependencies before importing search,
+# mirroring test_search_pagination.py so the module imports in isolation.
+_fake_query = types.ModuleType("rag.nlp.query")
+
+
+class _DummyFulltextQueryer:
+ pass
+
+
+_fake_query.FulltextQueryer = _DummyFulltextQueryer
+
+_STUBS = {
+ "rag.nlp.query": _fake_query,
+ "rag.nlp.rag_tokenizer": types.ModuleType("rag.nlp.rag_tokenizer"),
+ "common.settings": types.ModuleType("common.settings"),
+}
+
+# Install the stubs only for the import below, then take them back out. Leaving
+# them in sys.modules is process-global and breaks any later test that imports
+# the real module, for example test/unit_test/common/test_settings_queue.py.
+_previous = {name: sys.modules.get(name) for name in _STUBS}
+for _name, _module in _STUBS.items():
+ sys.modules.setdefault(_name, _module)
+
+try:
+ from rag.nlp.search import Dealer # noqa: E402
+finally:
+ for _name, _prior in _previous.items():
+ if _prior is not None:
+ sys.modules[_name] = _prior
+ elif sys.modules.get(_name) is _STUBS[_name]:
+ del sys.modules[_name]
+
+
+def _dealer():
+ dealer = Dealer.__new__(Dealer)
+ dealer._doc_exists_cache = OrderedDict()
+ dealer._doc_exists_lock = threading.Lock()
+ return dealer
+
+
+def _stub_document_service(monkeypatch, existing_ids):
+ """Stand in for DocumentService.get_by_ids(...).dicts(), which the method imports lazily."""
+ calls = []
+
+ class _Rows:
+ def __init__(self, ids):
+ self._ids = ids
+
+ def dicts(self):
+ return [{"id": doc_id} for doc_id in self._ids]
+
+ class _DocumentService:
+ @staticmethod
+ def get_by_ids(doc_ids):
+ calls.append(list(doc_ids))
+ return _Rows([d for d in doc_ids if d in existing_ids])
+
+ module = types.ModuleType("api.db.services.document_service")
+ module.DocumentService = _DocumentService
+ monkeypatch.setitem(sys.modules, "api.db.services.document_service", module)
+ return calls
+
+
+async def test_deleted_doc_stays_absent_on_the_cached_second_call(monkeypatch):
+ """A doc proven deleted must not come back as existing while the cache is warm."""
+ dealer = _dealer()
+ calls = _stub_document_service(monkeypatch, existing_ids=set())
+
+ first = await dealer._existing_doc_ids(["deleted-doc"])
+ second = await dealer._existing_doc_ids(["deleted-doc"])
+
+ assert first == set()
+ assert second == set()
+ # The point of caching the negative result: no second database round-trip.
+ assert len(calls) == 1
+
+
+async def test_existing_doc_is_served_from_cache_without_a_second_query(monkeypatch):
+ dealer = _dealer()
+ calls = _stub_document_service(monkeypatch, existing_ids={"live-doc"})
+
+ first = await dealer._existing_doc_ids(["live-doc"])
+ second = await dealer._existing_doc_ids(["live-doc"])
+
+ assert first == {"live-doc"}
+ assert second == {"live-doc"}
+ assert len(calls) == 1
+
+
+async def test_mixed_batch_keeps_only_the_live_doc_when_cached(monkeypatch):
+ dealer = _dealer()
+ calls = _stub_document_service(monkeypatch, existing_ids={"live-doc"})
+
+ await dealer._existing_doc_ids(["live-doc", "deleted-doc"])
+ second = await dealer._existing_doc_ids(["live-doc", "deleted-doc"])
+
+ assert second == {"live-doc"}
+ # A regression that re-queries would still return {"live-doc"}, so pin the
+ # round-trip count too.
+ assert len(calls) == 1