Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(document): hide a disabled document's knowledge-compiled chunks from retrieval (#19845)

qinling0210bd58e961d7aa
diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.go
index d33af456f5..0987e351bc 100644
--- a/internal/engine/elasticsearch/chunk_helpers_test.go
+++ b/internal/engine/elasticsearch/chunk_helpers_test.go
@@ -250,10 +250,11 @@ func TestDeleteChunksIDStringSlice(t *testing.T) {
}
// TestUpdateChunksPreservesStringSliceCondition guards the document
-// availability switch: updateSourceChunkAvailability passes a typed []string id
-// list, and a builder that only understands []interface{} silently drops the id
-// clause, widening the update-by-query to every chunk of the dataset (kb_id is
-// then the only remaining filter).
+// availability switch: the doc-service caller of UpdateChunks
+// (updateDocumentChunkAvailability) passes a typed []string id list, and a
+// builder that only understands []interface{} silently drops the id clause,
+// widening the update-by-query to every chunk of the dataset (kb_id is then the
+// only remaining filter).
func TestUpdateChunksPreservesStringSliceCondition(t *testing.T) {
var updateQuery map[string]interface{}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
diff --git a/internal/engine/infinity/sql_test.go b/internal/engine/infinity/sql_test.go
index 232e77e464..feaa07c98d 100644
--- a/internal/engine/infinity/sql_test.go
+++ b/internal/engine/infinity/sql_test.go
@@ -402,10 +402,11 @@ func TestBuildFilterFromCondition_UnconstrainedFilter(t *testing.T) {
}
// TestBuildFilterFromCondition_StringSliceIDPreservesScope pins the shape the
-// document availability switch relies on: updateSourceChunkAvailability passes a
-// typed []string id list, and Infinity must render it as an IN clause. Dropping
-// it leaves whatever other clauses the caller passed (none, for that path), i.e.
-// an update scoped to the whole dataset table.
+// document availability switch relies on: the doc-service caller of
+// UpdateChunks (updateDocumentChunkAvailability) passes a typed []string id
+// list, and Infinity must render it as an IN clause. Dropping it leaves whatever
+// other clauses the caller passed (none, for that path), i.e. an update scoped
+// to the whole dataset table.
func TestBuildFilterFromCondition_StringSliceIDPreservesScope(t *testing.T) {
clmns := map[string]struct {
Type string
diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.go
index 7fdc536f04..771fd9cfe0 100644
--- a/internal/service/document/document_dataset_update.go
+++ b/internal/service/document/document_dataset_update.go
@@ -80,7 +80,7 @@ func (s *DocumentService) BatchUpdateDocumentStatus(ctx context.Context, userID,
hasError = true
continue
}
- err = s.updateSourceChunkAvailability(ctx, kb.TenantID, doc.KbID, docID, statusInt)
+ err = s.updateDocumentChunkAvailability(ctx, kb.TenantID, doc.KbID, docID, statusInt)
if err != nil {
_ = s.documentDAO.UpdateByID(ctx, dao.DB, docID, map[string]interface{}{"status": previousStatus})
msg := err.Error()
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index a3e91bd7dc..e67b18cd13 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -584,7 +584,7 @@ func (s *DocumentService) updateDocumentStatusOnly(ctx context.Context, doc *ent
}
if s.docEngine != nil {
- if err := s.updateSourceChunkAvailability(ctx, kb.TenantID, doc.KbID, doc.ID, status); err != nil {
+ if err := s.updateDocumentChunkAvailability(ctx, kb.TenantID, doc.KbID, doc.ID, status); err != nil {
return err
}
}
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index f233be260b..1db7aec67a 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -295,18 +295,25 @@ func (e *failingDeleteDocEngine) DeleteChunks(context.Context, map[string]interf
type sourceAvailabilityDocEngine struct {
fakeChatDocEngine
updateConditions []map[string]interface{}
+ updateValues []map[string]interface{}
}
func (e *sourceAvailabilityDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {
return &types.SearchResult{Chunks: []map[string]interface{}{
{"id": "source-1"},
+ {"id": "tree-1", "compile_kwd": "tree"},
+ // structure stamps its inferred kind verbatim; both are final products.
+ {"id": "struct-1", "compile_kwd": "hypergraph"},
+ {"id": "pageindex-1", "compile_kwd": "page_index"},
{"id": "wiki-1", "compile_kwd": "wiki_page"},
{"id": "map-1", "compile_kwd": []interface{}{"wiki_map_active"}},
- }, Total: 3}, nil
+ {"id": "nav-1", "compile_kwd": "dataset_nav"},
+ }, Total: 7}, nil
}
-func (e *sourceAvailabilityDocEngine) UpdateChunks(_ context.Context, condition map[string]interface{}, _ map[string]interface{}, _, _ string) error {
+func (e *sourceAvailabilityDocEngine) UpdateChunks(_ context.Context, condition map[string]interface{}, newValue map[string]interface{}, _, _ string) error {
e.updateConditions = append(e.updateConditions, condition)
+ e.updateValues = append(e.updateValues, newValue)
return nil
}
@@ -2542,20 +2549,27 @@ func TestClearDocumentParseResultsPurgesTaskStateBeforeDeletingTask(t *testing.T
}
}
-func TestUpdateSourceChunkAvailabilityExcludesCompiledProducts(t *testing.T) {
+// TestUpdateDocumentChunkAvailabilityTogglesFinalProducts pins the disable/enable
+// contract: source chunks and final compiled products (tree/structure/mindmap)
+// follow the document status; wiki staging and unknown-kwd rows stay hidden.
+func TestUpdateDocumentChunkAvailabilityTogglesFinalProducts(t *testing.T) {
docEngine := &sourceAvailabilityDocEngine{}
svc := testDocumentService(t)
svc.docEngine = docEngine
- if err := svc.updateSourceChunkAvailability(t.Context(), "tenant-1", "kb-1", "doc-1", 1); err != nil {
- t.Fatalf("updateSourceChunkAvailability failed: %v", err)
+ if err := svc.updateDocumentChunkAvailability(t.Context(), "tenant-1", "kb-1", "doc-1", 0); err != nil {
+ t.Fatalf("updateDocumentChunkAvailability failed: %v", err)
}
if len(docEngine.updateConditions) != 1 {
t.Fatalf("UpdateChunks calls = %d, want 1", len(docEngine.updateConditions))
}
ids, ok := docEngine.updateConditions[0]["id"].([]string)
- if !ok || len(ids) != 1 || ids[0] != "source-1" {
- t.Fatalf("updated ids = %#v, want only source-1", docEngine.updateConditions[0]["id"])
+ want := []string{"source-1", "tree-1", "struct-1", "pageindex-1"}
+ if !ok || !reflect.DeepEqual(ids, want) {
+ t.Fatalf("updated ids = %#v, want %v (wiki/unknown-kwd rows excluded)", docEngine.updateConditions[0]["id"], want)
+ }
+ if got := docEngine.updateValues[0]["available_int"]; got != 0 {
+ t.Fatalf("available_int = %#v, want 0", got)
}
}
diff --git a/internal/service/document/wiki_refresh.go b/internal/service/document/wiki_refresh.go
index 06805e8f46..af320f7a38 100644
--- a/internal/service/document/wiki_refresh.go
+++ b/internal/service/document/wiki_refresh.go
@@ -23,6 +23,7 @@ import (
"ragflow/internal/common"
enginetypes "ragflow/internal/engine/types"
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/ingestion/knowledge_compile"
"go.uber.org/zap"
@@ -30,15 +31,17 @@ import (
const sourceChunkAvailabilityBatchSize = 1000
-// updateSourceChunkAvailability changes retrieval visibility for source chunks
-// only. Document-level compiler products must remain hidden (available_int=0)
-// until the dataset-level compiler has merged them.
-func (s *DocumentService) updateSourceChunkAvailability(ctx context.Context, tenantID, datasetID, documentID string, available int) error {
+// updateDocumentChunkAvailability changes retrieval visibility for every row a
+// disabled/enabled document is allowed to expose: its ordinary source chunks and
+// its per-document final compiled products (tree / structure / mindmap — the wiki
+// variant is hidden at write time, see markCompiledProductsHidden). Wiki staging
+// rows and unknown-kwd rows are skipped; see loadAvailabilityToggleChunkIDs.
+func (s *DocumentService) updateDocumentChunkAvailability(ctx context.Context, tenantID, datasetID, documentID string, available int) error {
if s.docEngine == nil {
return fmt.Errorf("document engine not initialized")
}
indexName := fmt.Sprintf("ragflow_%s", tenantID)
- ids, err := s.loadSourceChunkIDs(ctx, indexName, datasetID, documentID)
+ ids, err := s.loadAvailabilityToggleChunkIDs(ctx, indexName, datasetID, documentID)
if err != nil {
return err
}
@@ -56,6 +59,49 @@ func (s *DocumentService) updateSourceChunkAvailability(ctx context.Context, ten
return nil
}
+// loadAvailabilityToggleChunkIDs returns the document's own rows whose
+// availability follows the document status: source chunks (no compile_kwd) and
+// final compiled products (a compile_kwd mapping to a non-wiki variant). Skipped:
+// the wiki variant (wiki_page / wiki_section), whose per-document rows are
+// staging for the dataset-level merge, and unknown-kwd rows such as the
+// wiki_map_active state row or the legacy, KB-scoped wiki_page_graph blob.
+func (s *DocumentService) loadAvailabilityToggleChunkIDs(ctx context.Context, indexName, datasetID, documentID string) ([]string, error) {
+ ids := make([]string, 0)
+ for offset := 0; ; offset += sourceChunkAvailabilityBatchSize {
+ searchCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)
+ result, err := s.docEngine.Search(searchCtx, &enginetypes.SearchRequest{
+ IndexNames: []string{indexName},
+ KbIDs: []string{datasetID},
+ Offset: offset,
+ Limit: sourceChunkAvailabilityBatchSize,
+ SelectFields: []string{"id", "compile_kwd"},
+ Filter: map[string]any{"doc_id": []string{documentID}},
+ })
+ cancel()
+ if err != nil {
+ return nil, err
+ }
+ if result == nil || len(result.Chunks) == 0 {
+ break
+ }
+ for _, row := range result.Chunks {
+ if kwd := strings.TrimSpace(documentStoreString(row["compile_kwd"])); kwd != "" {
+ variant, variantErr := knowledge_compile.KwdToVariant(kwd)
+ if variantErr != nil || variant == kccommon.VariantWiki {
+ continue
+ }
+ }
+ if id := strings.TrimSpace(documentStoreString(row["id"])); id != "" {
+ ids = append(ids, id)
+ }
+ }
+ if int64(offset+len(result.Chunks)) >= result.Total {
+ break
+ }
+ }
+ return ids, nil
+}
+
func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, datasetID, documentID string) ([]string, error) {
ids := make([]string, 0)
for offset := 0; ; offset += sourceChunkAvailabilityBatchSize {