Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: exclude empty documents from structure alteration (#19246)

buua436f4bafa1596c9
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py
index 866324fb65..d8ba5d0d3b 100644
--- a/api/apps/services/dataset_api_service.py
+++ b/api/apps/services/dataset_api_service.py
@@ -2411,7 +2411,15 @@ def _eligible_doc_ids_for_kind(docs, tenant_id: str, kind: str) -> set:
return eligible
-async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, field: str | list[str], from_list: bool) -> set:
+async def _involved_doc_ids_paged(
+ index_nm,
+ dataset_id: str,
+ condition: dict,
+ field: str | list[str],
+ from_list: bool,
+ *,
+ raise_on_error: bool = False,
+) -> set:
"""Page a docStore search, folding provenance fields into a doc-id set.
``from_list`` reads the selected fields as lists of provenance document IDs;
@@ -2441,6 +2449,8 @@ async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, fi
rows = settings.docStoreConn.get_fields(res, select_fields) or {}
except Exception:
logging.exception("alteration: docStore search failed for kb=%s cond=%s", dataset_id, condition)
+ if raise_on_error:
+ raise
rows = {}
if not rows:
@@ -2459,6 +2469,24 @@ async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, fi
return involved
+async def _current_chunk_doc_ids(index_nm, dataset_id: str, doc_ids: set[str]) -> set[str]:
+ """Return eligible documents with available, not-yet-compiled source chunks."""
+ if not doc_ids:
+ return set()
+ return await _involved_doc_ids_paged(
+ index_nm,
+ dataset_id,
+ {
+ "doc_id": sorted(doc_ids),
+ "available_int": [1],
+ "must_not": {"exists": "compile_kwd"},
+ },
+ "doc_id",
+ from_list=False,
+ raise_on_error=True,
+ )
+
+
async def _involved_doc_ids_for_kind(index_nm, dataset_id: str, kind: str, tenant_id: str, wiki_map_state: dict | None = None) -> set:
"""Gather the doc ids baked into the compiled product for ``kind``."""
if kind == "wiki":
@@ -2551,6 +2579,17 @@ async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):
len(eligible_doc_ids),
len(current_chunk_state),
)
+ else:
+ eligible_before_chunk_filter = len(eligible_doc_ids)
+ chunk_doc_ids = await _current_chunk_doc_ids(index_nm, dataset_id, eligible_doc_ids)
+ eligible_doc_ids &= chunk_doc_ids
+ logging.debug(
+ "alteration: structure chunk eligibility kind=%s kb=%s before=%d after=%d",
+ kind,
+ dataset_id,
+ eligible_before_chunk_filter,
+ len(eligible_doc_ids),
+ )
wiki_map_state = None
if kind == "wiki":
from rag.advanced_rag.knowlege_compile.wiki import _wiki_load_active_map_state
@@ -2573,12 +2612,13 @@ async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):
current_chunk_state,
wiki_map_state,
)
- elif kind == "wiki":
- # Without the compiled source index there are no current chunks that
- # can be considered Wiki inputs.
+ else:
+ # Without the source index there are no current chunks that can be
+ # considered inputs for any structure kind.
eligible_doc_ids = set()
logging.debug(
- "alteration: Wiki compiled index missing kb=%s tenant=%s eligible=0",
+ "alteration: structure source index missing kind=%s kb=%s tenant=%s eligible=0",
+ kind,
dataset_id,
kb.tenant_id,
)
diff --git a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
index 2417a2e970..8e021165ef 100644
--- a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
+++ b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py
@@ -450,6 +450,59 @@ def test_wiki_alteration_treats_wiki_template_as_eligible(monkeypatch):
assert module._eligible_doc_ids_for_kind(docs, "tenant-1", "wiki") == {"doc-wiki"}
+async def test_structure_alteration_chunk_filter_excludes_unparsed_documents(monkeypatch):
+ module, _, _ = _load_list_datasets_module(
+ monkeypatch,
+ kbs=[],
+ parsing_status_by_kb={},
+ )
+
+ captured = {}
+
+ async def _paged(_index, dataset_id, condition, field, from_list, *, raise_on_error):
+ captured.update(dataset_id=dataset_id, condition=condition, field=field, from_list=from_list)
+ assert raise_on_error is True
+ return {"doc-with-chunk"}
+
+ monkeypatch.setattr(module, "_involved_doc_ids_paged", _paged)
+
+ result = await module._current_chunk_doc_ids(
+ "tenant-index",
+ "kb-1",
+ {"doc-with-chunk", "doc-without-chunk"},
+ )
+
+ assert result == {"doc-with-chunk"}
+ assert captured == {
+ "dataset_id": "kb-1",
+ "condition": {
+ "doc_id": ["doc-with-chunk", "doc-without-chunk"],
+ "available_int": [1],
+ "must_not": {"exists": "compile_kwd"},
+ },
+ "field": "doc_id",
+ "from_list": False,
+ }
+
+
+async def test_current_chunk_doc_ids_propagates_search_errors(monkeypatch):
+ module, _, _ = _load_list_datasets_module(
+ monkeypatch,
+ kbs=[],
+ parsing_status_by_kb={},
+ )
+
+ async def _paged(*_args, **_kwargs):
+ raise RuntimeError("search unavailable")
+
+ monkeypatch.setattr(module, "_involved_doc_ids_paged", _paged)
+
+ with pytest.raises(RuntimeError, match="search unavailable"):
+ await module._current_chunk_doc_ids("tenant-index", "kb-1", {"doc-1"})
+
+
@pytest.mark.asyncio
async def test_wiki_involved_ids_use_active_map_state_when_pages_are_missing(monkeypatch):
"""Use MAP provenance even when a participating document has no page row."""