RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(document): hide a disabled document's knowledge-compiled chunks from retrieval (#19845)
qinling0210bd58e961d7aa
diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.goindex d33af456f5..0987e351bc 100644--- a/internal/engine/elasticsearch/chunk_helpers_test.go+++ b/internal/engine/elasticsearch/chunk_helpers_test.go@@ -250,10 +250,11 @@ func TestDeleteChunksIDStringSlice(t *testing.T) {}// TestUpdateChunksPreservesStringSliceCondition guards the document-// availability switch: updateSourceChunkAvailability passes a typed []string id-// list, and a builder that only understands []interface{} silently drops the id-// clause, widening the update-by-query to every chunk of the dataset (kb_id is-// then the only remaining filter).+// availability switch: the doc-service caller of UpdateChunks+// (updateDocumentChunkAvailability) passes a typed []string id list, and a+// builder that only understands []interface{} silently drops the id clause,+// widening the update-by-query to every chunk of the dataset (kb_id is then the+// only remaining filter).func TestUpdateChunksPreservesStringSliceCondition(t *testing.T) {var updateQuery map[string]interface{}server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {diff --git a/internal/engine/infinity/sql_test.go b/internal/engine/infinity/sql_test.goindex 232e77e464..feaa07c98d 100644--- a/internal/engine/infinity/sql_test.go+++ b/internal/engine/infinity/sql_test.go@@ -402,10 +402,11 @@ func TestBuildFilterFromCondition_UnconstrainedFilter(t *testing.T) {}// TestBuildFilterFromCondition_StringSliceIDPreservesScope pins the shape the-// document availability switch relies on: updateSourceChunkAvailability passes a-// typed []string id list, and Infinity must render it as an IN clause. Dropping-// it leaves whatever other clauses the caller passed (none, for that path), i.e.-// an update scoped to the whole dataset table.+// document availability switch relies on: the doc-service caller of+// UpdateChunks (updateDocumentChunkAvailability) passes a typed []string id+// list, and Infinity must render it as an IN clause. Dropping it leaves whatever+// other clauses the caller passed (none, for that path), i.e. an update scoped+// to the whole dataset table.func TestBuildFilterFromCondition_StringSliceIDPreservesScope(t *testing.T) {clmns := map[string]struct {Type stringdiff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.goindex 7fdc536f04..771fd9cfe0 100644--- a/internal/service/document/document_dataset_update.go+++ b/internal/service/document/document_dataset_update.go@@ -80,7 +80,7 @@ func (s *DocumentService) BatchUpdateDocumentStatus(ctx context.Context, userID,hasError = truecontinue}- err = s.updateSourceChunkAvailability(ctx, kb.TenantID, doc.KbID, docID, statusInt)+ err = s.updateDocumentChunkAvailability(ctx, kb.TenantID, doc.KbID, docID, statusInt)if err != nil {_ = s.documentDAO.UpdateByID(ctx, dao.DB, docID, map[string]interface{}{"status": previousStatus})msg := err.Error()diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.goindex a3e91bd7dc..e67b18cd13 100644--- a/internal/service/document/document_parse.go+++ b/internal/service/document/document_parse.go@@ -584,7 +584,7 @@ func (s *DocumentService) updateDocumentStatusOnly(ctx context.Context, doc *ent}if s.docEngine != nil {- if err := s.updateSourceChunkAvailability(ctx, kb.TenantID, doc.KbID, doc.ID, status); err != nil {+ if err := s.updateDocumentChunkAvailability(ctx, kb.TenantID, doc.KbID, doc.ID, status); err != nil {return err}}diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex f233be260b..1db7aec67a 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -295,18 +295,25 @@ func (e *failingDeleteDocEngine) DeleteChunks(context.Context, map[string]interftype sourceAvailabilityDocEngine struct {fakeChatDocEngineupdateConditions []map[string]interface{}+ updateValues []map[string]interface{}}func (e *sourceAvailabilityDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {return &types.SearchResult{Chunks: []map[string]interface{}{{"id": "source-1"},+ {"id": "tree-1", "compile_kwd": "tree"},+ // structure stamps its inferred kind verbatim; both are final products.+ {"id": "struct-1", "compile_kwd": "hypergraph"},+ {"id": "pageindex-1", "compile_kwd": "page_index"},{"id": "wiki-1", "compile_kwd": "wiki_page"},{"id": "map-1", "compile_kwd": []interface{}{"wiki_map_active"}},- }, Total: 3}, nil+ {"id": "nav-1", "compile_kwd": "dataset_nav"},+ }, Total: 7}, nil}-func (e *sourceAvailabilityDocEngine) UpdateChunks(_ context.Context, condition map[string]interface{}, _ map[string]interface{}, _, _ string) error {+func (e *sourceAvailabilityDocEngine) UpdateChunks(_ context.Context, condition map[string]interface{}, newValue map[string]interface{}, _, _ string) error {e.updateConditions = append(e.updateConditions, condition)+ e.updateValues = append(e.updateValues, newValue)return nil}@@ -2542,20 +2549,27 @@ func TestClearDocumentParseResultsPurgesTaskStateBeforeDeletingTask(t *testing.T}}-func TestUpdateSourceChunkAvailabilityExcludesCompiledProducts(t *testing.T) {+// TestUpdateDocumentChunkAvailabilityTogglesFinalProducts pins the disable/enable+// contract: source chunks and final compiled products (tree/structure/mindmap)+// follow the document status; wiki staging and unknown-kwd rows stay hidden.+func TestUpdateDocumentChunkAvailabilityTogglesFinalProducts(t *testing.T) {docEngine := &sourceAvailabilityDocEngine{}svc := testDocumentService(t)svc.docEngine = docEngine- if err := svc.updateSourceChunkAvailability(t.Context(), "tenant-1", "kb-1", "doc-1", 1); err != nil {- t.Fatalf("updateSourceChunkAvailability failed: %v", err)+ if err := svc.updateDocumentChunkAvailability(t.Context(), "tenant-1", "kb-1", "doc-1", 0); err != nil {+ t.Fatalf("updateDocumentChunkAvailability failed: %v", err)}if len(docEngine.updateConditions) != 1 {t.Fatalf("UpdateChunks calls = %d, want 1", len(docEngine.updateConditions))}ids, ok := docEngine.updateConditions[0]["id"].([]string)- if !ok || len(ids) != 1 || ids[0] != "source-1" {- t.Fatalf("updated ids = %#v, want only source-1", docEngine.updateConditions[0]["id"])+ want := []string{"source-1", "tree-1", "struct-1", "pageindex-1"}+ if !ok || !reflect.DeepEqual(ids, want) {+ t.Fatalf("updated ids = %#v, want %v (wiki/unknown-kwd rows excluded)", docEngine.updateConditions[0]["id"], want)+ }+ if got := docEngine.updateValues[0]["available_int"]; got != 0 {+ t.Fatalf("available_int = %#v, want 0", got)}}diff --git a/internal/service/document/wiki_refresh.go b/internal/service/document/wiki_refresh.goindex 06805e8f46..af320f7a38 100644--- a/internal/service/document/wiki_refresh.go+++ b/internal/service/document/wiki_refresh.go@@ -23,6 +23,7 @@ import ("ragflow/internal/common"enginetypes "ragflow/internal/engine/types"+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/ingestion/knowledge_compile""go.uber.org/zap"@@ -30,15 +31,17 @@ import (const sourceChunkAvailabilityBatchSize = 1000-// updateSourceChunkAvailability changes retrieval visibility for source chunks-// only. Document-level compiler products must remain hidden (available_int=0)-// until the dataset-level compiler has merged them.-func (s *DocumentService) updateSourceChunkAvailability(ctx context.Context, tenantID, datasetID, documentID string, available int) error {+// updateDocumentChunkAvailability changes retrieval visibility for every row a+// disabled/enabled document is allowed to expose: its ordinary source chunks and+// its per-document final compiled products (tree / structure / mindmap — the wiki+// variant is hidden at write time, see markCompiledProductsHidden). Wiki staging+// rows and unknown-kwd rows are skipped; see loadAvailabilityToggleChunkIDs.+func (s *DocumentService) updateDocumentChunkAvailability(ctx context.Context, tenantID, datasetID, documentID string, available int) error {if s.docEngine == nil {return fmt.Errorf("document engine not initialized")}indexName := fmt.Sprintf("ragflow_%s", tenantID)- ids, err := s.loadSourceChunkIDs(ctx, indexName, datasetID, documentID)+ ids, err := s.loadAvailabilityToggleChunkIDs(ctx, indexName, datasetID, documentID)if err != nil {return err}@@ -56,6 +59,49 @@ func (s *DocumentService) updateSourceChunkAvailability(ctx context.Context, tenreturn nil}+// loadAvailabilityToggleChunkIDs returns the document's own rows whose+// availability follows the document status: source chunks (no compile_kwd) and+// final compiled products (a compile_kwd mapping to a non-wiki variant). Skipped:+// the wiki variant (wiki_page / wiki_section), whose per-document rows are+// staging for the dataset-level merge, and unknown-kwd rows such as the+// wiki_map_active state row or the legacy, KB-scoped wiki_page_graph blob.+func (s *DocumentService) loadAvailabilityToggleChunkIDs(ctx context.Context, indexName, datasetID, documentID string) ([]string, error) {+ ids := make([]string, 0)+ for offset := 0; ; offset += sourceChunkAvailabilityBatchSize {+ searchCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)+ result, err := s.docEngine.Search(searchCtx, &enginetypes.SearchRequest{+ IndexNames: []string{indexName},+ KbIDs: []string{datasetID},+ Offset: offset,+ Limit: sourceChunkAvailabilityBatchSize,+ SelectFields: []string{"id", "compile_kwd"},+ Filter: map[string]any{"doc_id": []string{documentID}},+ })+ cancel()+ if err != nil {+ return nil, err+ }+ if result == nil || len(result.Chunks) == 0 {+ break+ }+ for _, row := range result.Chunks {+ if kwd := strings.TrimSpace(documentStoreString(row["compile_kwd"])); kwd != "" {+ variant, variantErr := knowledge_compile.KwdToVariant(kwd)+ if variantErr != nil || variant == kccommon.VariantWiki {+ continue+ }+ }+ if id := strings.TrimSpace(documentStoreString(row["id"])); id != "" {+ ids = append(ids, id)+ }+ }+ if int64(offset+len(result.Chunks)) >= result.Total {+ break+ }+ }+ return ids, nil+}+func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, datasetID, documentID string) ([]string, error) {ids := make([]string, 0)for offset := 0; ; offset += sourceChunkAvailabilityBatchSize {