Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: inherit document status when inserting chunks (#18610)

天海蒼灆bed9cc5a4f72
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 358b4f3524..46b497d400 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -260,8 +260,21 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
// KnowledgeCompiler component and stamped with `compile_kwd`) are persisted
// as available_int=0: invisible to the normal retriever until the dataset-level
// post-processing consumer (§11) merges them into available_int=1 products.
- // Ordinary source chunks stay available_int=1 (the index default).
+ // Ordinary source chunks stay available_int=1 (the index default) unless the
+ // document itself is disabled (status=0).
markCompiledProductsHidden(chunks)
+ // Reload status immediately before write: LoadFromIngestionTask copied a
+ // snapshot that may be stale if BatchUpdateDocumentStatus ran mid-pipeline.
+ docStatus := s.taskCtx.Doc.Status
+ if dao.DB != nil {
+ if persisted, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, s.taskCtx.Doc.ID); err == nil && persisted != nil {
+ docStatus = persisted.Status
+ s.taskCtx.Doc.Status = persisted.Status
+ } else if err != nil {
+ common.Warn(fmt.Sprintf("failed to reload document %s status before availability stamp: %v", s.taskCtx.Doc.ID, err))
+ }
+ }
+ applyDocumentAvailability(chunks, docStatus)
oldCompiledProductIDs, oldCompiledVariants, err := s.loadDocumentCompiledState(ctx)
if err != nil {
@@ -509,6 +522,22 @@ func (s *PipelineExecutor) reconcileDocumentCompiledProducts(ctx context.Context
return nil
}
+// applyDocumentAvailability stamps ordinary source chunks with available_int=0
+// when Document.status is "0" (disabled). Disabling before any chunks exist only
+// updates MySQL; without this, later parsing would still write searchable
+// available_int=1 rows. Compiled products (compile_kwd) stay at 0 regardless.
+func applyDocumentAvailability(chunks []map[string]any, status *string) {
+ if status == nil || *status != "0" {
+ return
+ }
+ for _, ck := range chunks {
+ if _, ok := ck["compile_kwd"]; ok {
+ continue
+ }
+ ck["available_int"] = 0
+ }
+}
+
// compiledVariants returns the sorted, de-duplicated set of compile types a
// document's compiled products carry. It reads the authoritative
// `compilation_template_kind_kwd` the KnowledgeCompiler component stamps on each
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index c0716f0333..ecae6a6c08 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -67,6 +67,40 @@ func TestWikiActiveStatesDecodeCheckpointValues(t *testing.T) {
}
}
+// TestApplyDocumentAvailability verifies disabled documents (status=0) force
+// ordinary source chunks to available_int=0 while compiled products stay hidden.
+func TestApplyDocumentAvailability(t *testing.T) {
+ chunks := []map[string]any{
+ {"id": "src-1", "content_with_weight": "ordinary source chunk"},
+ {"id": "struct-1", "compile_kwd": "structure", "content_with_weight": "entity A", "available_int": 0},
+ {"id": "src-2", "content_with_weight": "another source chunk"},
+ }
+ markCompiledProductsHidden(chunks)
+ applyDocumentAvailability(chunks, strPtr("0"))
+
+ if chunks[0]["available_int"] != 0 {
+ t.Fatalf("disabled doc source chunk should be available_int=0, got %v", chunks[0]["available_int"])
+ }
+ if chunks[1]["available_int"] != 0 {
+ t.Fatalf("compiled product should stay available_int=0, got %v", chunks[1]["available_int"])
+ }
+ if chunks[2]["available_int"] != 0 {
+ t.Fatalf("disabled doc source chunk should be available_int=0, got %v", chunks[2]["available_int"])
+ }
+
+ enabled := []map[string]any{
+ {"id": "src-3", "content_with_weight": "enabled source"},
+ }
+ applyDocumentAvailability(enabled, strPtr("1"))
+ if v, ok := enabled[0]["available_int"]; ok {
+ t.Fatalf("enabled doc should keep default available_int, got %v", v)
+ }
+ applyDocumentAvailability(enabled, nil)
+ if v, ok := enabled[0]["available_int"]; ok {
+ t.Fatalf("nil status should keep default available_int, got %v", v)
+ }
+}
+
func makeTaskCtx() *TaskContext {
return &TaskContext{
IngestionTask: &entity.IngestionTask{
diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py
index 7e1c397bfc..aada32fc38 100644
--- a/rag/svr/task_executor.py
+++ b/rag/svr/task_executor.py
@@ -1300,6 +1300,10 @@ async def insert_chunks(task_id, task_tenant_id, task_dataset_id, chunks, progre
chunks: List of chunk dictionaries to insert
progress_callback: Callback function for progress updates
"""
+ from rag.svr.task_executor_refactor.chunk_service import apply_source_chunks_document_availability
+
+ apply_source_chunks_document_availability(chunks)
+
mothers = []
mother_ids = set([])
for ck in chunks:
diff --git a/rag/svr/task_executor_refactor/chunk_service.py b/rag/svr/task_executor_refactor/chunk_service.py
index bf5e3d3016..d792c353cd 100644
--- a/rag/svr/task_executor_refactor/chunk_service.py
+++ b/rag/svr/task_executor_refactor/chunk_service.py
@@ -38,13 +38,13 @@ from common.connection_utils import timeout
from common.constants import PAGERANK_FLD, TAG_FLD
from common.misc_utils import thread_pool_exec
from common.float_utils import normalize_overlapped_percent
+from api.db.services.document_service import DocumentService
+from api.db.services.task_service import TaskService
from rag.nlp import search
+from rag.svr.task_executor_refactor.constants import GRAPH_RAPTOR_FAKE_DOC_ID
from rag.svr.task_executor_refactor.task_context import TaskContext
from rag.utils.base64_image import image2id
-from api.db.services.task_service import TaskService
-from rag.svr.task_executor_refactor.constants import GRAPH_RAPTOR_FAKE_DOC_ID
-
# Re-export for backward compatibility
from rag.svr.task_executor_refactor.chunk_builder import (
get_parser,
@@ -60,6 +60,52 @@ from rag.svr.task_executor_refactor.chunk_post_processor import (
)
+def apply_document_availability(chunks: List[Dict[str, Any]], status) -> int:
+ """Stamp ordinary source chunks available_int=0 when document status is disabled.
+
+ Compiled products (compile_kwd) are left alone — they are already non-searchable.
+ Returns the number of ordinary source chunks stamped.
+ """
+ if str(status if status is not None else "1") != "0":
+ return 0
+ stamped = 0
+ for ck in chunks:
+ if ck.get("compile_kwd"):
+ continue
+ ck["available_int"] = 0
+ stamped += 1
+ return stamped
+
+
+def apply_source_chunks_document_availability(chunks: List[Dict[str, Any]]) -> None:
+ """Apply each source document's status to its ordinary (non-RAPTOR) chunks.
+
+ Mixed RAPTOR batches can carry chunks from multiple documents; stamp each
+ group from its own document status instead of the first doc_id in the batch.
+ """
+ source_chunks_by_doc_id: Dict[str, List[Dict[str, Any]]] = {}
+ for ck in chunks:
+ if ck.get("raptor_kwd"):
+ continue
+ doc_id = ck.get("doc_id")
+ if not doc_id or doc_id == GRAPH_RAPTOR_FAKE_DOC_ID:
+ continue
+ source_chunks_by_doc_id.setdefault(doc_id, []).append(ck)
+
+ for doc_id, source_chunks in source_chunks_by_doc_id.items():
+ ok, doc = DocumentService.get_by_id(doc_id)
+ if not ok or doc is None:
+ continue
+ status = getattr(doc, "status", "1")
+ stamped = apply_document_availability(source_chunks, status)
+ if stamped:
+ logging.info(
+ "document %s is disabled; stamped available_int=0 on %d ordinary source chunk(s)",
+ doc_id,
+ stamped,
+ )
+
+
class ChunkService:
"""Service for document chunking and post-processing.
@@ -259,6 +305,8 @@ class ChunkService:
"""
doc_bulk_size = doc_bulk_size or settings.DOC_BULK_SIZE
+ self._apply_document_availability(chunks)
+
# Create mother chunks (summary chunks)
mothers = self._create_mother_chunks(chunks)
@@ -269,6 +317,26 @@ class ChunkService:
# Insert main chunks
return await self._insert_main_chunks(task_id, task_tenant_id, task_dataset_id, chunks, doc_bulk_size)
+ def _apply_document_availability(self, chunks: List[Dict[str, Any]]) -> None:
+ """Hide source chunks when the owning document is disabled (status=0).
+
+ Disabling before parse only updates MySQL; later inserts would otherwise
+ default to available_int=1 and remain retrievable.
+ """
+ doc_id = self._task_context.doc_id
+ if not doc_id or doc_id == GRAPH_RAPTOR_FAKE_DOC_ID:
+ return
+ ok, doc = DocumentService.get_by_id(doc_id)
+ if not ok or doc is None:
+ return
+ stamped = apply_document_availability(chunks, getattr(doc, "status", "1"))
+ if stamped:
+ logging.info(
+ "document %s is disabled; stamped available_int=0 on %d ordinary source chunk(s)",
+ doc_id,
+ stamped,
+ )
+
@classmethod
def _create_mother_chunks(cls, chunks: List[Dict]) -> List[Dict]:
"""Create mother chunks from summary fields.
diff --git a/test/unit_test/rag/svr/task_executor_refactor/test_document_availability.py b/test/unit_test/rag/svr/task_executor_refactor/test_document_availability.py
new file mode 100644
index 0000000000..51d2a21db6
--- /dev/null
+++ b/test/unit_test/rag/svr/task_executor_refactor/test_document_availability.py
@@ -0,0 +1,71 @@
+#
+# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from rag.svr.task_executor_refactor.chunk_service import (
+ apply_document_availability,
+ apply_source_chunks_document_availability,
+)
+from rag.svr.task_executor_refactor.constants import GRAPH_RAPTOR_FAKE_DOC_ID
+
+
+def test_apply_document_availability_disabled_hides_source_chunks():
+ chunks = [
+ {"id": "src-1", "content_with_weight": "ordinary source"},
+ {"id": "struct-1", "compile_kwd": "structure", "available_int": 0},
+ {"id": "src-2", "content_with_weight": "another source"},
+ ]
+ stamped = apply_document_availability(chunks, "0")
+
+ assert stamped == 2
+ assert chunks[0]["available_int"] == 0
+ assert chunks[1]["available_int"] == 0
+ assert chunks[2]["available_int"] == 0
+
+
+def test_apply_document_availability_enabled_keeps_default():
+ chunks = [{"id": "src-1", "content_with_weight": "ordinary source"}]
+ assert apply_document_availability(chunks, "1") == 0
+ assert "available_int" not in chunks[0]
+
+ assert apply_document_availability(chunks, None) == 0
+ assert "available_int" not in chunks[0]
+
+
+def test_apply_source_chunks_skips_raptor_and_stamps_per_doc():
+ """Mixed RAPTOR batches must not inherit status from the first document only."""
+ chunks = [
+ {"id": "raptor-1", "doc_id": "disabled-doc", "raptor_kwd": "raptor"},
+ {"id": "src-disabled", "doc_id": "disabled-doc", "content_with_weight": "from disabled"},
+ {"id": "src-enabled", "doc_id": "enabled-doc", "content_with_weight": "from enabled"},
+ {"id": "fake", "doc_id": GRAPH_RAPTOR_FAKE_DOC_ID, "content_with_weight": "fake raptor doc"},
+ ]
+
+ def _get_by_id(doc_id):
+ status = {"disabled-doc": "0", "enabled-doc": "1"}.get(doc_id, "1")
+ return True, SimpleNamespace(status=status)
+
+ with patch(
+ "rag.svr.task_executor_refactor.chunk_service.DocumentService.get_by_id",
+ side_effect=_get_by_id,
+ ):
+ apply_source_chunks_document_availability(chunks)
+
+ assert "available_int" not in chunks[0] # RAPTOR chunk untouched
+ assert chunks[1]["available_int"] == 0 # disabled source stamped
+ assert "available_int" not in chunks[2] # enabled source untouched
+ assert "available_int" not in chunks[3] # fake RAPTOR doc skipped