RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: inherit document status when inserting chunks (#18610)
天海蒼灆bed9cc5a4f72
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 358b4f3524..46b497d400 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -260,8 +260,21 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map// KnowledgeCompiler component and stamped with `compile_kwd`) are persisted// as available_int=0: invisible to the normal retriever until the dataset-level// post-processing consumer (§11) merges them into available_int=1 products.- // Ordinary source chunks stay available_int=1 (the index default).+ // Ordinary source chunks stay available_int=1 (the index default) unless the+ // document itself is disabled (status=0).markCompiledProductsHidden(chunks)+ // Reload status immediately before write: LoadFromIngestionTask copied a+ // snapshot that may be stale if BatchUpdateDocumentStatus ran mid-pipeline.+ docStatus := s.taskCtx.Doc.Status+ if dao.DB != nil {+ if persisted, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, s.taskCtx.Doc.ID); err == nil && persisted != nil {+ docStatus = persisted.Status+ s.taskCtx.Doc.Status = persisted.Status+ } else if err != nil {+ common.Warn(fmt.Sprintf("failed to reload document %s status before availability stamp: %v", s.taskCtx.Doc.ID, err))+ }+ }+ applyDocumentAvailability(chunks, docStatus)oldCompiledProductIDs, oldCompiledVariants, err := s.loadDocumentCompiledState(ctx)if err != nil {@@ -509,6 +522,22 @@ func (s *PipelineExecutor) reconcileDocumentCompiledProducts(ctx context.Contextreturn nil}+// applyDocumentAvailability stamps ordinary source chunks with available_int=0+// when Document.status is "0" (disabled). Disabling before any chunks exist only+// updates MySQL; without this, later parsing would still write searchable+// available_int=1 rows. Compiled products (compile_kwd) stay at 0 regardless.+func applyDocumentAvailability(chunks []map[string]any, status *string) {+ if status == nil || *status != "0" {+ return+ }+ for _, ck := range chunks {+ if _, ok := ck["compile_kwd"]; ok {+ continue+ }+ ck["available_int"] = 0+ }+}+// compiledVariants returns the sorted, de-duplicated set of compile types a// document's compiled products carry. It reads the authoritative// `compilation_template_kind_kwd` the KnowledgeCompiler component stamps on eachdiff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex c0716f0333..ecae6a6c08 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -67,6 +67,40 @@ func TestWikiActiveStatesDecodeCheckpointValues(t *testing.T) {}}+// TestApplyDocumentAvailability verifies disabled documents (status=0) force+// ordinary source chunks to available_int=0 while compiled products stay hidden.+func TestApplyDocumentAvailability(t *testing.T) {+ chunks := []map[string]any{+ {"id": "src-1", "content_with_weight": "ordinary source chunk"},+ {"id": "struct-1", "compile_kwd": "structure", "content_with_weight": "entity A", "available_int": 0},+ {"id": "src-2", "content_with_weight": "another source chunk"},+ }+ markCompiledProductsHidden(chunks)+ applyDocumentAvailability(chunks, strPtr("0"))++ if chunks[0]["available_int"] != 0 {+ t.Fatalf("disabled doc source chunk should be available_int=0, got %v", chunks[0]["available_int"])+ }+ if chunks[1]["available_int"] != 0 {+ t.Fatalf("compiled product should stay available_int=0, got %v", chunks[1]["available_int"])+ }+ if chunks[2]["available_int"] != 0 {+ t.Fatalf("disabled doc source chunk should be available_int=0, got %v", chunks[2]["available_int"])+ }++ enabled := []map[string]any{+ {"id": "src-3", "content_with_weight": "enabled source"},+ }+ applyDocumentAvailability(enabled, strPtr("1"))+ if v, ok := enabled[0]["available_int"]; ok {+ t.Fatalf("enabled doc should keep default available_int, got %v", v)+ }+ applyDocumentAvailability(enabled, nil)+ if v, ok := enabled[0]["available_int"]; ok {+ t.Fatalf("nil status should keep default available_int, got %v", v)+ }+}+func makeTaskCtx() *TaskContext {return &TaskContext{IngestionTask: &entity.IngestionTask{diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.pyindex 7e1c397bfc..aada32fc38 100644--- a/rag/svr/task_executor.py+++ b/rag/svr/task_executor.py@@ -1300,6 +1300,10 @@ async def insert_chunks(task_id, task_tenant_id, task_dataset_id, chunks, progrechunks: List of chunk dictionaries to insertprogress_callback: Callback function for progress updates"""+ from rag.svr.task_executor_refactor.chunk_service import apply_source_chunks_document_availability++ apply_source_chunks_document_availability(chunks)+mothers = []mother_ids = set([])for ck in chunks:diff --git a/rag/svr/task_executor_refactor/chunk_service.py b/rag/svr/task_executor_refactor/chunk_service.pyindex bf5e3d3016..d792c353cd 100644--- a/rag/svr/task_executor_refactor/chunk_service.py+++ b/rag/svr/task_executor_refactor/chunk_service.py@@ -38,13 +38,13 @@ from common.connection_utils import timeoutfrom common.constants import PAGERANK_FLD, TAG_FLDfrom common.misc_utils import thread_pool_execfrom common.float_utils import normalize_overlapped_percent+from api.db.services.document_service import DocumentService+from api.db.services.task_service import TaskServicefrom rag.nlp import search+from rag.svr.task_executor_refactor.constants import GRAPH_RAPTOR_FAKE_DOC_IDfrom rag.svr.task_executor_refactor.task_context import TaskContextfrom rag.utils.base64_image import image2id-from api.db.services.task_service import TaskService-from rag.svr.task_executor_refactor.constants import GRAPH_RAPTOR_FAKE_DOC_ID-# Re-export for backward compatibilityfrom rag.svr.task_executor_refactor.chunk_builder import (get_parser,@@ -60,6 +60,52 @@ from rag.svr.task_executor_refactor.chunk_post_processor import ()+def apply_document_availability(chunks: List[Dict[str, Any]], status) -> int:+ """Stamp ordinary source chunks available_int=0 when document status is disabled.++ Compiled products (compile_kwd) are left alone — they are already non-searchable.+ Returns the number of ordinary source chunks stamped.+ """+ if str(status if status is not None else "1") != "0":+ return 0+ stamped = 0+ for ck in chunks:+ if ck.get("compile_kwd"):+ continue+ ck["available_int"] = 0+ stamped += 1+ return stamped+++def apply_source_chunks_document_availability(chunks: List[Dict[str, Any]]) -> None:+ """Apply each source document's status to its ordinary (non-RAPTOR) chunks.++ Mixed RAPTOR batches can carry chunks from multiple documents; stamp each+ group from its own document status instead of the first doc_id in the batch.+ """+ source_chunks_by_doc_id: Dict[str, List[Dict[str, Any]]] = {}+ for ck in chunks:+ if ck.get("raptor_kwd"):+ continue+ doc_id = ck.get("doc_id")+ if not doc_id or doc_id == GRAPH_RAPTOR_FAKE_DOC_ID:+ continue+ source_chunks_by_doc_id.setdefault(doc_id, []).append(ck)++ for doc_id, source_chunks in source_chunks_by_doc_id.items():+ ok, doc = DocumentService.get_by_id(doc_id)+ if not ok or doc is None:+ continue+ status = getattr(doc, "status", "1")+ stamped = apply_document_availability(source_chunks, status)+ if stamped:+ logging.info(+ "document %s is disabled; stamped available_int=0 on %d ordinary source chunk(s)",+ doc_id,+ stamped,+ )++class ChunkService:"""Service for document chunking and post-processing.@@ -259,6 +305,8 @@ class ChunkService:"""doc_bulk_size = doc_bulk_size or settings.DOC_BULK_SIZE+ self._apply_document_availability(chunks)+# Create mother chunks (summary chunks)mothers = self._create_mother_chunks(chunks)@@ -269,6 +317,26 @@ class ChunkService:# Insert main chunksreturn await self._insert_main_chunks(task_id, task_tenant_id, task_dataset_id, chunks, doc_bulk_size)+ def _apply_document_availability(self, chunks: List[Dict[str, Any]]) -> None:+ """Hide source chunks when the owning document is disabled (status=0).++ Disabling before parse only updates MySQL; later inserts would otherwise+ default to available_int=1 and remain retrievable.+ """+ doc_id = self._task_context.doc_id+ if not doc_id or doc_id == GRAPH_RAPTOR_FAKE_DOC_ID:+ return+ ok, doc = DocumentService.get_by_id(doc_id)+ if not ok or doc is None:+ return+ stamped = apply_document_availability(chunks, getattr(doc, "status", "1"))+ if stamped:+ logging.info(+ "document %s is disabled; stamped available_int=0 on %d ordinary source chunk(s)",+ doc_id,+ stamped,+ )+@classmethoddef _create_mother_chunks(cls, chunks: List[Dict]) -> List[Dict]:"""Create mother chunks from summary fields.diff --git a/test/unit_test/rag/svr/task_executor_refactor/test_document_availability.py b/test/unit_test/rag/svr/task_executor_refactor/test_document_availability.pynew file mode 100644index 0000000000..51d2a21db6--- /dev/null+++ b/test/unit_test/rag/svr/task_executor_refactor/test_document_availability.py@@ -0,0 +1,71 @@+#+# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from types import SimpleNamespace+from unittest.mock import patch++from rag.svr.task_executor_refactor.chunk_service import (+ apply_document_availability,+ apply_source_chunks_document_availability,+)+from rag.svr.task_executor_refactor.constants import GRAPH_RAPTOR_FAKE_DOC_ID+++def test_apply_document_availability_disabled_hides_source_chunks():+ chunks = [+ {"id": "src-1", "content_with_weight": "ordinary source"},+ {"id": "struct-1", "compile_kwd": "structure", "available_int": 0},+ {"id": "src-2", "content_with_weight": "another source"},+ ]+ stamped = apply_document_availability(chunks, "0")++ assert stamped == 2+ assert chunks[0]["available_int"] == 0+ assert chunks[1]["available_int"] == 0+ assert chunks[2]["available_int"] == 0+++def test_apply_document_availability_enabled_keeps_default():+ chunks = [{"id": "src-1", "content_with_weight": "ordinary source"}]+ assert apply_document_availability(chunks, "1") == 0+ assert "available_int" not in chunks[0]++ assert apply_document_availability(chunks, None) == 0+ assert "available_int" not in chunks[0]+++def test_apply_source_chunks_skips_raptor_and_stamps_per_doc():+ """Mixed RAPTOR batches must not inherit status from the first document only."""+ chunks = [+ {"id": "raptor-1", "doc_id": "disabled-doc", "raptor_kwd": "raptor"},+ {"id": "src-disabled", "doc_id": "disabled-doc", "content_with_weight": "from disabled"},+ {"id": "src-enabled", "doc_id": "enabled-doc", "content_with_weight": "from enabled"},+ {"id": "fake", "doc_id": GRAPH_RAPTOR_FAKE_DOC_ID, "content_with_weight": "fake raptor doc"},+ ]++ def _get_by_id(doc_id):+ status = {"disabled-doc": "0", "enabled-doc": "1"}.get(doc_id, "1")+ return True, SimpleNamespace(status=status)++ with patch(+ "rag.svr.task_executor_refactor.chunk_service.DocumentService.get_by_id",+ side_effect=_get_by_id,+ ):+ apply_source_chunks_document_availability(chunks)++ assert "available_int" not in chunks[0] # RAPTOR chunk untouched+ assert chunks[1]["available_int"] == 0 # disabled source stamped+ assert "available_int" not in chunks[2] # enabled source untouched+ assert "available_int" not in chunks[3] # fake RAPTOR doc skipped