RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: exclude empty documents from structure alteration (#19246)
buua436f4bafa1596c9
diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex 866324fb65..d8ba5d0d3b 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -2411,7 +2411,15 @@ def _eligible_doc_ids_for_kind(docs, tenant_id: str, kind: str) -> set:return eligible-async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, field: str | list[str], from_list: bool) -> set:+async def _involved_doc_ids_paged(+ index_nm,+ dataset_id: str,+ condition: dict,+ field: str | list[str],+ from_list: bool,+ *,+ raise_on_error: bool = False,+) -> set:"""Page a docStore search, folding provenance fields into a doc-id set.``from_list`` reads the selected fields as lists of provenance document IDs;@@ -2441,6 +2449,8 @@ async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, firows = settings.docStoreConn.get_fields(res, select_fields) or {}except Exception:logging.exception("alteration: docStore search failed for kb=%s cond=%s", dataset_id, condition)+ if raise_on_error:+ raiserows = {}if not rows:@@ -2459,6 +2469,24 @@ async def _involved_doc_ids_paged(index_nm, dataset_id: str, condition: dict, fireturn involved+async def _current_chunk_doc_ids(index_nm, dataset_id: str, doc_ids: set[str]) -> set[str]:+ """Return eligible documents with available, not-yet-compiled source chunks."""+ if not doc_ids:+ return set()+ return await _involved_doc_ids_paged(+ index_nm,+ dataset_id,+ {+ "doc_id": sorted(doc_ids),+ "available_int": [1],+ "must_not": {"exists": "compile_kwd"},+ },+ "doc_id",+ from_list=False,+ raise_on_error=True,+ )++async def _involved_doc_ids_for_kind(index_nm, dataset_id: str, kind: str, tenant_id: str, wiki_map_state: dict | None = None) -> set:"""Gather the doc ids baked into the compiled product for ``kind``."""if kind == "wiki":@@ -2551,6 +2579,17 @@ async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):len(eligible_doc_ids),len(current_chunk_state),)+ else:+ eligible_before_chunk_filter = len(eligible_doc_ids)+ chunk_doc_ids = await _current_chunk_doc_ids(index_nm, dataset_id, eligible_doc_ids)+ eligible_doc_ids &= chunk_doc_ids+ logging.debug(+ "alteration: structure chunk eligibility kind=%s kb=%s before=%d after=%d",+ kind,+ dataset_id,+ eligible_before_chunk_filter,+ len(eligible_doc_ids),+ )wiki_map_state = Noneif kind == "wiki":from rag.advanced_rag.knowlege_compile.wiki import _wiki_load_active_map_state@@ -2573,12 +2612,13 @@ async def _get_alteration(dataset_id: str, tenant_id: str, kind: str):current_chunk_state,wiki_map_state,)- elif kind == "wiki":- # Without the compiled source index there are no current chunks that- # can be considered Wiki inputs.+ else:+ # Without the source index there are no current chunks that can be+ # considered inputs for any structure kind.eligible_doc_ids = set()logging.debug(- "alteration: Wiki compiled index missing kb=%s tenant=%s eligible=0",+ "alteration: structure source index missing kind=%s kb=%s tenant=%s eligible=0",+ kind,dataset_id,kb.tenant_id,)diff --git a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.pyindex 2417a2e970..8e021165ef 100644--- a/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py+++ b/test/unit_test/api/apps/services/test_dataset_api_service_list_datasets.py@@ -450,6 +450,59 @@ def test_wiki_alteration_treats_wiki_template_as_eligible(monkeypatch):assert module._eligible_doc_ids_for_kind(docs, "tenant-1", "wiki") == {"doc-wiki"}+async def test_structure_alteration_chunk_filter_excludes_unparsed_documents(monkeypatch):+ module, _, _ = _load_list_datasets_module(+ monkeypatch,+ kbs=[],+ parsing_status_by_kb={},+ )++ captured = {}++ async def _paged(_index, dataset_id, condition, field, from_list, *, raise_on_error):+ captured.update(dataset_id=dataset_id, condition=condition, field=field, from_list=from_list)+ assert raise_on_error is True+ return {"doc-with-chunk"}++ monkeypatch.setattr(module, "_involved_doc_ids_paged", _paged)++ result = await module._current_chunk_doc_ids(+ "tenant-index",+ "kb-1",+ {"doc-with-chunk", "doc-without-chunk"},+ )++ assert result == {"doc-with-chunk"}+ assert captured == {+ "dataset_id": "kb-1",+ "condition": {+ "doc_id": ["doc-with-chunk", "doc-without-chunk"],+ "available_int": [1],+ "must_not": {"exists": "compile_kwd"},+ },+ "field": "doc_id",+ "from_list": False,+ }+++async def test_current_chunk_doc_ids_propagates_search_errors(monkeypatch):+ module, _, _ = _load_list_datasets_module(+ monkeypatch,+ kbs=[],+ parsing_status_by_kb={},+ )++ async def _paged(*_args, **_kwargs):+ raise RuntimeError("search unavailable")++ monkeypatch.setattr(module, "_involved_doc_ids_paged", _paged)++ with pytest.raises(RuntimeError, match="search unavailable"):+ await module._current_chunk_doc_ids("tenant-index", "kb-1", {"doc-1"})++@pytest.mark.asyncioasync def test_wiki_involved_ids_use_active_map_state_when_pages_are_missing(monkeypatch):"""Use MAP provenance even when a participating document has no page row."""