Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat: improve incremental Wiki compilation (#18557)

buua436a3df5889513e
diff --git a/internal/agent/runtime/helpers.go b/internal/agent/runtime/helpers.go
index c66c1f16ea..31e3b35b2d 100644
--- a/internal/agent/runtime/helpers.go
+++ b/internal/agent/runtime/helpers.go
@@ -88,6 +88,29 @@ type ProgressEvent struct {
// ProgressCallback receives progress notifications from TrackProgress.
type ProgressCallback func(event ProgressEvent)
+// ProgressMessageCallback receives detailed messages emitted by a component
+// while it is running. Unlike ProgressCallback, these messages do not change
+// the component lifecycle counters; they are supplemental observability for
+// stages such as a compiler's MAP/REDUCE/PLAN/REFINE pipeline.
+type ProgressMessageCallback func(component, message string)
+
+type progressMessageCallbackKey struct{}
+
+// WithProgressMessageCallback attaches a detailed component-message sink to a
+// run context. A nil callback is valid and keeps components DB-independent.
+func WithProgressMessageCallback(ctx context.Context, cb ProgressMessageCallback) context.Context {
+ return context.WithValue(ctx, progressMessageCallbackKey{}, cb)
+}
+
+// ReportProgressMessage forwards a detailed component message when a sink is
+// attached. Components can call this without depending on the persistence
+// layer or changing the lifecycle progress contract.
+func ReportProgressMessage(ctx context.Context, component, message string) {
+ if cb, ok := ctx.Value(progressMessageCallbackKey{}).(ProgressMessageCallback); ok && cb != nil {
+ cb(component, message)
+ }
+}
+
// TrackProgress wraps fn with progress notifications. The callback is
// invoked at most twice per call (once at start, once at end):
//
diff --git a/internal/dao/database.go b/internal/dao/database.go
index 818b0097d4..03f4efda1c 100644
--- a/internal/dao/database.go
+++ b/internal/dao/database.go
@@ -158,6 +158,7 @@ func InitDB(ctx context.Context, migrateDB bool) error {
&entity.FileCommit{},
&entity.FileCommitItem{},
&entity.KnowledgeCompileDataset{},
+ &entity.WikiDocumentDirty{},
// Knowledge-compile compilation templates and their groups. The Go
// KnowledgeCompilerComponent resolves a compilation_template (or group)
// from these tables at runtime, so the Go side must guarantee they exist.
diff --git a/internal/entity/dataset.go b/internal/entity/dataset.go
index 1c1bad0ddf..101662aeb0 100644
--- a/internal/entity/dataset.go
+++ b/internal/entity/dataset.go
@@ -82,6 +82,7 @@ const (
PipelineTaskTypeGraphRAG PipelineTaskType = "GraphRAG"
PipelineTaskTypeMindmap PipelineTaskType = "Mindmap"
PipelineTaskTypeMemory PipelineTaskType = "Memory"
+ PipelineTaskTypeWiki PipelineTaskType = "Wiki"
)
// FileSource represents the source of a file
diff --git a/internal/entity/knowledge_compile_doc.go b/internal/entity/knowledge_compile_doc.go
index 1c38685919..099c25d579 100644
--- a/internal/entity/knowledge_compile_doc.go
+++ b/internal/entity/knowledge_compile_doc.go
@@ -61,7 +61,10 @@ type KnowledgeCompileDataset struct {
// ErrorMsg is the most recent failure/retry diagnostic. It is TEXT with no
// DDL default (MySQL 8.0.13+ rejects a literal default on TEXT, Error 1101);
// the application always writes it explicitly when set.
- ErrorMsg string `gorm:"column:error_msg;type:text;not null" json:"error_msg"`
+ ErrorMsg string `gorm:"column:error_msg;type:text;not null" json:"error_msg"`
+ Progress float64 `gorm:"column:progress;not null;default:0" json:"progress"`
+ CurrentPhase string `gorm:"column:current_phase;size:64;not null;default:''" json:"current_phase"`
+ ProgressMsg string `gorm:"column:progress_msg;type:text;not null" json:"progress_msg"`
// LastCompletedAt records the last time the backlog drained to empty.
LastCompletedAt *time.Time `gorm:"column:last_completed_at;default:null" json:"last_completed_at"`
CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"created_at"`
diff --git a/internal/entity/wiki_document_dirty.go b/internal/entity/wiki_document_dirty.go
new file mode 100644
index 0000000000..eff002de59
--- /dev/null
+++ b/internal/entity/wiki_document_dirty.go
@@ -0,0 +1,43 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package entity
+
+import "time"
+
+const (
+ WikiDirtyStatePending = "pending"
+ WikiDirtyStateRunning = "running"
+)
+
+// WikiDocumentDirty is the durable trailing-edge debounce row for direct chunk
+// mutations. Each document owns one row; another mutation increments Revision,
+// merges the affected chunk ids, and moves RunAfter twenty seconds forward.
+type WikiDocumentDirty struct {
+ DocumentID string `gorm:"primaryKey;column:document_id;size:64" json:"document_id"`
+ DatasetID string `gorm:"column:dataset_id;size:64;not null;index" json:"dataset_id"`
+ TenantID string `gorm:"column:tenant_id;size:64;not null;index" json:"tenant_id"`
+ Revision uint64 `gorm:"column:revision;not null;default:1" json:"revision"`
+ AffectedChunkIDs string `gorm:"column:affected_chunk_ids;type:text;not null" json:"affected_chunk_ids"`
+ RunAfter time.Time `gorm:"column:run_after;not null;index" json:"run_after"`
+ State string `gorm:"column:state;size:16;not null;default:'pending';index" json:"state"`
+ ClaimOwner string `gorm:"column:claim_owner;size:64;not null;default:''" json:"claim_owner"`
+ ClaimExpiresAt *time.Time `gorm:"column:claim_expires_at;default:null;index" json:"claim_expires_at"`
+ ErrorMsg string `gorm:"column:error_msg;type:text;not null" json:"error_msg"`
+ CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"created_at"`
+ UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime" json:"updated_at"`
+}
+
+func (WikiDocumentDirty) TableName() string { return "wiki_document_dirty" }
diff --git a/internal/ingestion/component/knowledge_compiler/common/deps.go b/internal/ingestion/component/knowledge_compiler/common/deps.go
index 96111a0b41..5db40c801c 100644
--- a/internal/ingestion/component/knowledge_compiler/common/deps.go
+++ b/internal/ingestion/component/knowledge_compiler/common/deps.go
@@ -131,6 +131,24 @@ type WikiMapVersionStore interface {
PutWikiMapVersions(ctx context.Context, versions []WikiMapVersion) error
}
+// WikiMapActiveState is the mutable pointer to the chunk/hash MAP versions and
+// page plan used by the latest successful document Wiki compile. Payload is
+// owned by the wiki variant so the storage layer remains schema-independent.
+type WikiMapActiveState struct {
+ Key string
+ TenantID string
+ DatasetID string
+ DocumentID string
+ Payload []byte
+}
+
+// WikiMapActiveStateStore persists the active MAP/plan snapshot separately
+// from immutable WikiMapVersion history.
+type WikiMapActiveStateStore interface {
+ GetWikiMapActiveState(ctx context.Context, tenantID, datasetID, key string) ([]byte, error)
+ PutWikiMapActiveState(ctx context.Context, state WikiMapActiveState) error
+}
+
// RedisClient is injected only for the datasetnav variant (M8).
type RedisClient interface {
// Minimal lock surface; full API added in M8.
diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.go
index 31445d54e6..6f803cb9ba 100644
--- a/internal/ingestion/component/knowledge_compiler/common/types.go
+++ b/internal/ingestion/component/knowledge_compiler/common/types.go
@@ -138,6 +138,9 @@ type Product struct {
type Outputs struct {
Products []Product
DuplicatesDropped int
+ AffectedPageSlugs []string
+ RemovedPageSlugs []string
+ WikiActiveStates []WikiMapActiveState
}
// ParseParam builds a Param from the DSL params map. The variant is NOT taken
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index 29696603cd..05e3a07978 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -210,6 +210,9 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
o.Products[i].Variant = variant
}
out.Products = append(out.Products, o.Products...)
+ out.AffectedPageSlugs = append(out.AffectedPageSlugs, o.AffectedPageSlugs...)
+ out.RemovedPageSlugs = append(out.RemovedPageSlugs, o.RemovedPageSlugs...)
+ out.WikiActiveStates = append(out.WikiActiveStates, o.WikiActiveStates...)
}
// Convert the compiled products into chunk-aligned docs (matching
@@ -246,7 +249,52 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
zap.Int("wiki_relation", relationCount),
zap.Int("chunk_docs", len(compiled)),
)
- return mergeChunks(inputs, compiled), nil
+ result := mergeChunks(inputs, compiled)
+ if len(out.AffectedPageSlugs) > 0 {
+ result["wiki_affected_slugs"] = uniqueSorted(out.AffectedPageSlugs)
+ }
+ if len(out.RemovedPageSlugs) > 0 {
+ result["wiki_removed_slugs"] = uniqueSorted(out.RemovedPageSlugs)
+ }
+ if len(out.WikiActiveStates) > 0 {
+ result["wiki_active_map_states"] = wikiActiveStateValues(out.WikiActiveStates)
+ }
+ return result, nil
+}
+
+// wikiActiveStateValues keeps the component output checkpoint-safe. Pipeline
+// node outputs cross an eino serialization boundary, so package-specific Go
+// structs must not escape in map[string]any values.
+func wikiActiveStateValues(states []common.WikiMapActiveState) []map[string]any {
+ values := make([]map[string]any, 0, len(states))
+ for _, state := range states {
+ values = append(values, map[string]any{
+ "key": state.Key,
+ "tenant_id": state.TenantID,
+ "dataset_id": state.DatasetID,
+ "document_id": state.DocumentID,
+ "payload": string(state.Payload),
+ })
+ }
+ return values
+}
+
+func uniqueSorted(values []string) []string {
+ seen := make(map[string]struct{}, len(values))
+ result := make([]string, 0, len(values))
+ for _, value := range values {
+ value = strings.TrimSpace(value)
+ if value == "" {
+ continue
+ }
+ if _, ok := seen[value]; ok {
+ continue
+ }
+ seen[value] = struct{}{}
+ result = append(result, value)
+ }
+ sort.Strings(result)
+ return result
}
// resolveTemplateSpecs resolves the configured compilation template spec(s) to
diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.go
index 21420a5805..b5369a9e15 100644
--- a/internal/ingestion/component/knowledge_compiler/component_test.go
+++ b/internal/ingestion/component/knowledge_compiler/component_test.go
@@ -19,6 +19,18 @@ import (
"gorm.io/gorm"
)
+func TestWikiActiveStateValuesAreCheckpointSafe(t *testing.T) {
+ values := wikiActiveStateValues([]common.WikiMapActiveState{{
+ Key: "state-1", TenantID: "tenant-1", DatasetID: "kb-1", DocumentID: "doc-1", Payload: []byte(`{"plan":[]}`),
+ }})
+ if len(values) != 1 || values[0]["payload"] != `{"plan":[]}` {
+ t.Fatalf("checkpoint values = %#v", values)
+ }
+ if _, err := json.Marshal(map[string]any{"wiki_active_map_states": values}); err != nil {
+ t.Fatalf("marshal checkpoint-safe active states: %v", err)
+ }
+}
+
// mockChat answers the structure variant's three LLM call shapes under the
// Python-aligned items contract: node extraction (entity items), edge
// extraction (relation items constrained to Known Entities), and merge
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
index de9db1975c..717cbac419 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
@@ -17,7 +17,9 @@ import (
"regexp"
"sort"
"strings"
+ "sync"
+ "ragflow/internal/agent/runtime"
appcommon "ragflow/internal/common"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/ingestion/component/knowledge_compiler/structure"
@@ -61,6 +63,8 @@ func runBatches(ctx context.Context, jobs []func() error) error {
// hitting the output-token limit and truncating the payload.
const wikiMapTokenBudget = 2048
+const wikiRefineProgressStep = 5
+
// wikiMapMaxTokens derives the extraction output budget from the model's
// context length and the per-batch input budget: once the batch has consumed
// wikiMapTokenBudget input tokens, the rest of the window is handed to the
@@ -105,6 +109,15 @@ type wikiPipeline struct {
// hard cap; testable and reported for observability.
planCapacityExcluded int
neighborCache map[string]*common.WikiPageCandidate
+ incremental bool
+ activeStateKey string
+ previousActiveState wikiMapActiveSnapshot
+ nextActiveState wikiMapActiveSnapshot
+ affectedPageSlugs map[string]struct{}
+ removedPageSlugs []string
+ mapChanged bool
+ affectedTerms map[string]struct{}
+ pendingActiveState *common.WikiMapActiveState
}
type wikiExtract struct {
@@ -116,6 +129,16 @@ type wikiExtract struct {
Mode string `json:"mode,omitempty"`
}
+type wikiMapActiveSnapshot struct {
+ Chunks map[string]wikiMapActiveChunk `json:"chunks"`
+ Plan []wikiPlanPage `json:"plan"`
+}
+
+type wikiMapActiveChunk struct {
+ Key string `json:"key"`
+ Extract wikiExtract `json:"extract"`
+}
+
type wikiEntity struct {
Name string `json:"name"`
Type string `json:"type"`
@@ -219,16 +242,21 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
zap.Int("chunks", len(inputs.Chunks)),
zap.Bool("chat_ready", deps.Chat != nil),
zap.Bool("embed_ready", deps.Embed != nil))
+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("Wiki Started: input_chunks=%d", len(inputs.Chunks)))
p := &wikiPipeline{
- ctx: ctx,
- deps: deps,
- param: param,
- inputs: inputs,
- docID: docID,
- tenantID: deps.TenantID,
- datasetID: deps.DatasetID,
- llmID: llmID,
- neighborCache: make(map[string]*common.WikiPageCandidate),
+ ctx: ctx,
+ deps: deps,
+ param: param,
+ inputs: inputs,
+ docID: docID,
+ tenantID: deps.TenantID,
+ datasetID: deps.DatasetID,
+ llmID: llmID,
+ neighborCache: make(map[string]*common.WikiPageCandidate),
+ affectedPageSlugs: make(map[string]struct{}),
+ }
+ if incremental, ok := inputs.VariantSpecific["wiki_incremental"].(bool); ok {
+ p.incremental = incremental
}
if err := p.run(); err != nil {
appcommon.Error("wiki: Run pipeline failed", err,
@@ -244,6 +272,8 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
zap.Int("refined_pages", len(p.pages)))
products := buildWikiPageProducts(p.tenantID, p.docID, p.pages)
+ pageProducts, sectionProducts := countWikiProducts(products)
+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("Wiki Done: input_chunks=%d output_products=%d wiki_pages=%d wiki_sections=%d", len(inputs.Chunks), len(products), pageProducts, sectionProducts))
appcommon.Info("wiki: built page products",
zap.String("dataset_id", deps.DatasetID),
zap.String("doc_id", docID),
@@ -304,6 +334,11 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
out := common.Outputs{
Products: prod,
DuplicatesDropped: stats.DuplicatesDropped,
+ AffectedPageSlugs: sortedStringSet(p.affectedPageSlugs),
+ RemovedPageSlugs: append([]string(nil), p.removedPageSlugs...),
+ }
+ if p.pendingActiveState != nil {
+ out.WikiActiveStates = append(out.WikiActiveStates, *p.pendingActiveState)
}
return out, nil
}
@@ -325,8 +360,10 @@ func (p *wikiPipeline) runKey() string {
func (p *wikiPipeline) run() error {
if len(p.inputs.Chunks) == 0 {
+ runtime.ReportProgressMessage(p.ctx, "Compiler", "Wiki Empty: input_chunks=0 output_products=0")
return nil
}
+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki MAP Started: input_chunks=%d", len(p.inputs.Chunks)))
if err := p.runMap(); err != nil {
return err
}
@@ -335,6 +372,7 @@ func (p *wikiPipeline) run() error {
zap.String("doc_id", p.runKey()),
zap.Int("batches", len(p.mapExtracts)),
zap.Int("raw_entities", countExtractEntities(p.mapExtracts)))
+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki MAP Done: input_chunks=%d output_batches=%d entities=%d concepts=%d claims=%d relations=%d topics=%d", len(p.inputs.Chunks), len(p.mapExtracts), countExtractEntities(p.mapExtracts), countExtractConcepts(p.mapExtracts), countExtractClaims(p.mapExtracts), countExtractRelations(p.mapExtracts), countExtractTopics(p.mapExtracts)))
appcommon.Debug("wiki: MAP intermediate",
zap.String("dataset_id", p.datasetID),
zap.String("doc_id", p.runKey()),
@@ -343,6 +381,7 @@ func (p *wikiPipeline) run() error {
zap.Strings("concepts", extractConceptDebug(p.mapExtracts)),
zap.Strings("claims", extractClaimDebug(p.mapExtracts)),
zap.Strings("relations", extractRelationDebug(p.mapExtracts)))
+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REDUCE Started: input_entities=%d input_concepts=%d input_claims=%d input_relations=%d", countExtractEntities(p.mapExtracts), countExtractConcepts(p.mapExtracts), countExtractClaims(p.mapExtracts), countExtractRelations(p.mapExtracts)))
reduced := reduceExtracts(p.mapExtracts)
p.reduced = reduced
appcommon.Debug("wiki: REDUCE exact done",
@@ -370,12 +409,14 @@ func (p *wikiPipeline) run() error {
zap.String("doc_id", p.runKey()),
zap.Int("entities", len(p.reduced.Entities)),
zap.Int("claims", len(p.reduced.Claims)))
+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REDUCE Done: output_entities=%d output_concepts=%d output_claims=%d output_relations=%d", len(p.reduced.Entities), len(p.reduced.Concepts), len(p.reduced.Claims), len(p.reduced.Relations)))
// PLAN (B-mode only): LLM-based page plan + reconcile against existing pages.
// Mode A (param.PlanEnabled() == false, the default) skips the planner entirely — every
// extracted entity/concept becomes its own flat page (1 identity = 1 page), so
// the wiki is a flat encyclopedia without PLAN-grouped pages. See buildModeAPlan.
var plan wikiPlan
var err error
+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki PLAN Started: input_entities=%d input_concepts=%d input_relations=%d", len(p.reduced.Entities), len(p.reduced.Concepts), len(p.reduced.Relations)))
if p.param.PlanEnabled() {
plan, err = p.runPlan()
if err != nil {
@@ -389,11 +430,13 @@ func (p *wikiPipeline) run() error {
appcommon.Info("wiki: PLAN (A-mode flat) done", zap.String("dataset_id", p.datasetID), zap.String("doc_id", p.runKey()), zap.Int("plan_pages", len(plan.Pages)))
}
p.plan = plan
+ p.selectAffectedPages(plan.Pages)
appcommon.Info("wiki: PLAN done",
zap.String("dataset_id", p.datasetID),
zap.String("doc_id", p.runKey()),
zap.Int("plan_pages", len(plan.Pages)),
zap.Int("capacity_excluded", p.planCapacityExcluded))
+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki PLAN Done: output_pages=%d topics=%d", len(plan.Pages), countPlanTopics(plan.Pages)))
appcommon.Debug("wiki: PLAN intermediate",
zap.String("dataset_id", p.datasetID),
zap.String("doc_id", p.runKey()),
@@ -402,11 +445,20 @@ func (p *wikiPipeline) run() error {
zap.Int("plan_pages", len(plan.Pages)),
zap.Int("capacity_excluded", p.planCapacityExcluded),
zap.Strings("pages", planPagesDebug(plan.Pages)))
+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REFINE Started: input_pages=%d", len(p.plan.Pages)))
pages, err := p.runRefine()
if err != nil {
return err
}
p.pages = pages
+ activeState, err := p.buildActiveMapState(plan.Pages)
+ if err != nil {
+ return err
+ }
+ if activeState != nil {
+ p.pendingActiveState = activeState
+ }
+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REFINE Done: input_pages=%d output_pages=%d", len(p.plan.Pages), len(pages)))
appcommon.Info("wiki: REFINE done",
zap.String("dataset_id", p.datasetID),
zap.String("doc_id", p.runKey()),
@@ -428,6 +480,62 @@ func countExtractEntities(extracts []wikiExtract) int {
return n
}
+func countExtractConcepts(extracts []wikiExtract) int {
+ n := 0
+ for _, e := range extracts {
+ n += len(e.Concepts)
+ }
+ return n
+}
+
+func countExtractClaims(extracts []wikiExtract) int {
+ n := 0
+ for _, e := range extracts {
+ n += len(e.Claims)
+ }
+ return n
+}
+
+func countExtractRelations(extracts []wikiExtract) int {
+ n := 0
+ for _, e := range extracts {
+ n += len(e.Relations)
+ }
+ return n
+}
+
+func countExtractTopics(extracts []wikiExtract) int {
+ n := 0
+ for _, e := range extracts {
+ n += len(e.Topics)
+ }
+ return n
+}
+
+func countPlanTopics(pages []wikiPlanPage) int {
+ topics := make(map[string]struct{})
+ for _, page := range pages {
+ topic := strings.TrimSpace(page.Topic)
+ if topic != "" {
+ topics[topic] = struct{}{}
+ }
+ }
+ return len(topics)
+}
+
+func countWikiProducts(products []common.Product) (pages, sections int) {
+ for _, product := range products {
+ kind, _ := product.Meta["kind"].(string)
+ switch kind {
+ case "page":
+ pages++
+ case "section":
+ sections++
+ }
+ }
+ return pages, sections
+}
+
// --- Debug helpers: compact per-item descriptors for the stage-intermediate
// logs in run(). They deliberately record only identity-level fields (name /
// slug / type / aliases / endpoints), never full content or embedding vectors,
@@ -714,14 +822,23 @@ func (p *wikiPipeline) runLegacyPlan() (wikiPlan, error) {
// order regardless of completion order (no concurrent append to a shared slice).
// When no submitter is wired, jobs run serially (historic default).
func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {
- pages := normalizeWikiPlanPages(p.plan.Pages, p.reduced)
- if len(pages) == 0 {
+ allPages := normalizeWikiPlanPages(p.plan.Pages, p.reduced)
+ if len(allPages) == 0 {
return nil, nil
}
+ pages := allPages
+ if p.incremental {
+ pages = pages[:0:0]
+ for _, page := range allPages {
+ if _, affected := p.affectedPageSlugs[page.Slug]; affected {
+ pages = append(pages, page)
+ }
+ }
+ }
pageTitles := map[string]string{}
slugToPageType := map[string]string{}
- allPlanSlugs := make([]string, 0, len(pages))
- for _, page := range pages {
+ allPlanSlugs := make([]string, 0, len(allPages))
+ for _, page := range allPages {
if page.Slug == "" {
continue
}
@@ -739,6 +856,8 @@ func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {
results := make([]wikiPageResult, len(pages))
jobs := make([]func() error, 0, len(pages))
+ var progressMu sync.Mutex
+ completed := 0
for i, planItem := range pages {
i, planItem := i, planItem
jobs = append(jobs, func() error {
@@ -750,6 +869,12 @@ func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {
return err
}
results[i] = res
+ progressMu.Lock()
+ completed++
+ if shouldReportRefineProgress(completed, len(pages)) {
+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REFINE Progress: completed_pages=%d total_pages=%d", completed, len(pages)))
+ }
+ progressMu.Unlock()
return nil
})
}
@@ -759,6 +884,10 @@ func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {
return results, nil
}
+func shouldReportRefineProgress(completed, total int) bool {
+ return completed > 0 && total > 0 && (completed%wikiRefineProgressStep == 0 || completed == total)
+}
+
// runRefinePage generates one page result from a normalized plan page. UPDATE
// merge, evidence assembly, and source-context building are unchanged; this is
// the per-page unit that runRefine fans out.
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state.go
new file mode 100644
index 0000000000..aa497772c8
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state.go
@@ -0,0 +1,158 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package wiki
+
+import (
+ "encoding/json"
+ "fmt"
+ "sort"
+ "strings"
+
+ "ragflow/internal/ingestion/component/knowledge_compiler/common"
+)
+
+func (p *wikiPipeline) addAffectedExtractTerms(extract wikiExtract) {
+ if p.affectedTerms == nil {
+ p.affectedTerms = make(map[string]struct{})
+ }
+ add := func(value string) {
+ value = normalizedWikiTerm(value)
+ if value != "" {
+ p.affectedTerms[value] = struct{}{}
+ }
+ }
+ for _, entity := range extract.Entities {
+ add(entity.Name)
+ for _, alias := range entity.Aliases {
+ add(alias)
+ }
+ }
+ for _, concept := range extract.Concepts {
+ add(concept.Term)
+ }
+ for _, claim := range extract.Claims {
+ add(claim.Subject)
+ }
+ for _, relation := range extract.Relations {
+ add(relation.From)
+ add(relation.To)
+ }
+ for _, topic := range extract.Topics {
+ add(topic)
+ }
+}
+
+func (p *wikiPipeline) selectAffectedPages(current []wikiPlanPage) {
+ if p.affectedPageSlugs == nil {
+ p.affectedPageSlugs = make(map[string]struct{})
+ }
+ if !p.incremental || len(p.previousActiveState.Chunks) == 0 {
+ for _, page := range current {
+ if page.Slug != "" {
+ p.affectedPageSlugs[page.Slug] = struct{}{}
+ }
+ }
+ return
+ }
+ if !p.mapChanged {
+ return
+ }
+ previousBySlug := planPagesBySlug(p.previousActiveState.Plan)
+ currentBySlug := planPagesBySlug(current)
+ for slug, page := range currentBySlug {
+ previous, existed := previousBySlug[slug]
+ if !existed || planPageSignature(previous) != planPageSignature(page) || p.pageTouchesAffectedTerms(page) {
+ p.affectedPageSlugs[slug] = struct{}{}
+ }
+ }
+ for slug, page := range previousBySlug {
+ if _, remains := currentBySlug[slug]; !remains {
+ p.affectedPageSlugs[slug] = struct{}{}
+ p.removedPageSlugs = append(p.removedPageSlugs, slug)
+ continue
+ }
+ if p.pageTouchesAffectedTerms(page) {
+ p.affectedPageSlugs[slug] = struct{}{}
+ }
+ }
+ sort.Strings(p.removedPageSlugs)
+}
+
+func (p *wikiPipeline) pageTouchesAffectedTerms(page wikiPlanPage) bool {
+ if topic := normalizedWikiTerm(page.Topic); topic != "" {
+ if _, ok := p.affectedTerms[topic]; ok {
+ return true
+ }
+ }
+ for _, name := range page.EntityNames {
+ if _, ok := p.affectedTerms[normalizedWikiTerm(name)]; ok {
+ return true
+ }
+ }
+ return false
+}
+
+func (p *wikiPipeline) buildActiveMapState(plan []wikiPlanPage) (*common.WikiMapActiveState, error) {
+ if p.activeStateKey == "" || p.deps.WikiMapVersions == nil {
+ return nil, nil
+ }
+ p.nextActiveState.Plan = append([]wikiPlanPage(nil), plan...)
+ payload, err := json.Marshal(p.nextActiveState)
+ if err != nil {
+ return nil, fmt.Errorf("wiki: encode active MAP state: %w", err)
+ }
+ return &common.WikiMapActiveState{
+ Key: p.activeStateKey,
+ TenantID: p.tenantID,
+ DatasetID: p.datasetID,
+ DocumentID: p.docID,
+ Payload: payload,
+ }, nil
+}
+
+func planPagesBySlug(pages []wikiPlanPage) map[string]wikiPlanPage {
+ result := make(map[string]wikiPlanPage, len(pages))
+ for _, page := range pages {
+ if page.Slug != "" {
+ result[page.Slug] = page
+ }
+ }
+ return result
+}
+
+func planPageSignature(page wikiPlanPage) string {
+ payload, _ := json.Marshal(struct {
+ Title string
+ PageType string
+ Topic string
+ EntityNames []string
+ RelatedKB []string
+ }{page.Title, page.PageType, page.Topic, page.EntityNames, page.RelatedKB})
+ return string(payload)
+}
+
+func normalizedWikiTerm(value string) string {
+ return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(value)), " "))
+}
+
+func sortedStringSet(values map[string]struct{}) []string {
+ result := make([]string, 0, len(values))
+ for value := range values {
+ result = append(result, value)
+ }
+ sort.Strings(result)
+ return result
+}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state_test.go
new file mode 100644
index 0000000000..be68b0f81c
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state_test.go
@@ -0,0 +1,83 @@
+package wiki
+
+import "testing"
+
+func TestSelectAffectedPagesIncludesOldAndNewMembership(t *testing.T) {
+ pipeline := &wikiPipeline{
+ incremental: true,
+ mapChanged: true,
+ affectedTerms: map[string]struct{}{
+ "alpha": {},
+ },
+ affectedPageSlugs: map[string]struct{}{},
+ previousActiveState: wikiMapActiveSnapshot{
+ Chunks: map[string]wikiMapActiveChunk{"chunk-1": {Key: "old"}},
+ Plan: []wikiPlanPage{
+ {Slug: "topic/old", PageType: "topic", EntityNames: []string{"Alpha"}},
+ {Slug: "entity/stable", PageType: "entity", EntityNames: []string{"Stable"}},
+ },
+ },
+ }
+ current := []wikiPlanPage{
+ {Slug: "topic/new", PageType: "topic", EntityNames: []string{"Alpha"}},
+ {Slug: "entity/stable", PageType: "entity", EntityNames: []string{"Stable"}},
+ }
+
+ pipeline.selectAffectedPages(current)
+
+ if _, ok := pipeline.affectedPageSlugs["topic/old"]; !ok {
+ t.Fatal("old membership page was not affected")
+ }
+ if _, ok := pipeline.affectedPageSlugs["topic/new"]; !ok {
+ t.Fatal("new membership page was not affected")
+ }
+ if _, ok := pipeline.affectedPageSlugs["entity/stable"]; ok {
+ t.Fatal("unrelated stable page was affected")
+ }
+ if len(pipeline.removedPageSlugs) != 1 || pipeline.removedPageSlugs[0] != "topic/old" {
+ t.Fatalf("removed pages = %#v, want topic/old", pipeline.removedPageSlugs)
+ }
+}
+
+func TestSelectAffectedPagesNoChunkDeltaIsNoOp(t *testing.T) {
+ pipeline := &wikiPipeline{
+ incremental: true,
+ mapChanged: false,
+ affectedPageSlugs: map[string]struct{}{},
+ previousActiveState: wikiMapActiveSnapshot{
+ Chunks: map[string]wikiMapActiveChunk{"chunk-1": {Key: "same"}},
+ },
+ }
+ pipeline.selectAffectedPages([]wikiPlanPage{{Slug: "entity/alpha", EntityNames: []string{"Alpha"}}})
+ if len(pipeline.affectedPageSlugs) != 0 {
+ t.Fatalf("affected pages = %#v, want none", pipeline.affectedPageSlugs)
+ }
+}
+
+func TestSelectAffectedPagesMatchesOnlyChangedTopic(t *testing.T) {
+ pipeline := &wikiPipeline{
+ incremental: true,
+ mapChanged: true,
+ affectedTerms: map[string]struct{}{
+ "history": {},
+ },
+ affectedPageSlugs: map[string]struct{}{},
+ previousActiveState: wikiMapActiveSnapshot{
+ Chunks: map[string]wikiMapActiveChunk{"chunk-1": {Key: "old"}},
+ Plan: []wikiPlanPage{
+ {Slug: "topic/history", PageType: "topic", Topic: "History"},
+ {Slug: "topic/science", PageType: "topic", Topic: "Science"},
+ },
+ },
+ }
+ current := append([]wikiPlanPage(nil), pipeline.previousActiveState.Plan...)
+
+ pipeline.selectAffectedPages(current)
+
+ if _, ok := pipeline.affectedPageSlugs["topic/history"]; !ok {
+ t.Fatal("changed topic page was not selected")
+ }
+ if _, ok := pipeline.affectedPageSlugs["topic/science"]; ok {
+ t.Fatal("unrelated topic page was selected")
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go
index 47deab1bce..cf4eafbbcc 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go
@@ -42,6 +42,24 @@ func (p *wikiPipeline) runVersionedMap() error {
return err
}
llmFingerprint := wikiMapHash(p.llmID)
+ p.activeStateKey = wikiMapHash(strings.Join([]string{
+ p.tenantID, p.datasetID, p.docID, templateFingerprint, llmFingerprint, "active",
+ }, "\x00"))
+ p.previousActiveState = wikiMapActiveSnapshot{Chunks: map[string]wikiMapActiveChunk{}}
+ if activeStore, ok := p.deps.WikiMapVersions.(common.WikiMapActiveStateStore); ok {
+ payload, err := activeStore.GetWikiMapActiveState(p.ctx, p.tenantID, p.datasetID, p.activeStateKey)
+ if err != nil {
+ return fmt.Errorf("wiki: load active MAP state: %w", err)
+ }
+ if len(payload) > 0 {
+ if err := json.Unmarshal(payload, &p.previousActiveState); err != nil {
+ return fmt.Errorf("wiki: decode active MAP state: %w", err)
+ }
+ }
+ }
+ if p.previousActiveState.Chunks == nil {
+ p.previousActiveState.Chunks = map[string]wikiMapActiveChunk{}
+ }
versions := make([]wikiMapChunkVersion, 0, len(p.inputs.Chunks))
keys := make([]string, 0, len(p.inputs.Chunks))
@@ -149,6 +167,32 @@ func (p *wikiPipeline) runVersionedMap() error {
}
}
p.mapExtracts = append(p.mapExtracts, uncachedExtracts...)
+ p.nextActiveState = wikiMapActiveSnapshot{Chunks: make(map[string]wikiMapActiveChunk, len(versions))}
+ currentChunkIDs := make(map[string]struct{}, len(versions))
+ for _, version := range versions {
+ currentChunkIDs[version.chunk.ID] = struct{}{}
+ extract := extracts[version.index]
+ p.nextActiveState.Chunks[version.chunk.ID] = wikiMapActiveChunk{Key: version.key, Extract: extract}
+ previous, existed := p.previousActiveState.Chunks[version.chunk.ID]
+ if !existed || previous.Key != version.key {
+ p.mapChanged = true
+ p.addAffectedExtractTerms(previous.Extract)
+ p.addAffectedExtractTerms(extract)
+ }
+ }
+ for chunkID, previous := range p.previousActiveState.Chunks {
+ if _, active := currentChunkIDs[chunkID]; active {
+ continue
+ }
+ p.mapChanged = true
+ p.addAffectedExtractTerms(previous.Extract)
+ }
+ if len(uncacheable) > 0 {
+ p.mapChanged = true
+ for _, extract := range uncachedExtracts {
+ p.addAffectedExtractTerms(extract)
+ }
+ }
return nil
}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_refine_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_refine_test.go
index 67710cf020..af7b76166f 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_refine_test.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_refine_test.go
@@ -154,3 +154,12 @@ func TestRunRefine_CancelledCtxAborts(t *testing.T) {
t.Fatalf("runRefine err = nil, want context cancelled")
}
}
+
+func TestShouldReportRefineProgress(t *testing.T) {
+ for completed := 1; completed <= 12; completed++ {
+ want := completed == 5 || completed == 10 || completed == 12
+ if got := shouldReportRefineProgress(completed, 12); got != want {
+ t.Fatalf("completed=%d: got %t, want %t", completed, got, want)
+ }
+ }
+}
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index dcdcc5bcf2..1b09d1ec2a 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -344,6 +344,7 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
"output_format": "chunks",
"chunks": schema.ChunkDocsToMaps(chunks),
}
+ copyPipelineControlValues(out, inputs)
// Embedding requires a KB: the embedder (and its embd_id) is configured
// on the knowledgebase, so without kb_id there is nothing to resolve
@@ -369,6 +370,14 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
return out, nil
}
+func copyPipelineControlValues(output, input map[string]any) {
+ for _, key := range []string{"wiki_active_map_states"} {
+ if value, exists := input[key]; exists {
+ output[key] = value
+ }
+ }
+}
+
func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, embeddingModel, name string, chunks []schema.ChunkDoc) ([]schema.ChunkDoc, int, error) {
if len(chunks) == 0 {
return chunks, 0, nil
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index 985dd48981..5637323acd 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -24,6 +24,7 @@ import (
"context"
"encoding/json"
"os"
+ "reflect"
"strings"
"sync/atomic"
"testing"
@@ -264,6 +265,15 @@ func TestTokenizerComponent_InputsOutputs_NonEmpty(t *testing.T) {
}
}
+func TestCopyPipelineControlValuesPreservesWikiActiveState(t *testing.T) {
+ states := []map[string]any{{"key": "state-1", "payload": `{"plan":[]}`}}
+ output := map[string]any{"chunks": []any{}}
+ copyPipelineControlValues(output, map[string]any{"wiki_active_map_states": states})
+ if !reflect.DeepEqual(output["wiki_active_map_states"], states) {
+ t.Fatalf("wiki active states = %#v, want %#v", output["wiki_active_map_states"], states)
+ }
+}
+
// TestTokenizerComponent_NewTokenizerComponent_Defaults verifies
// the Python default param values propagate.
func TestTokenizerComponent_NewTokenizerComponent_Defaults(t *testing.T) {
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index ebac9d9d4a..ba55469cd9 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -47,10 +47,11 @@ var kcDB *gorm.DB
// broker — is the scheduling system of record and the source of same-KB
// serialization.
type Consumer struct {
- scheduler Claimer
- reader Reader
- writer Writer
- factory DeduperFactory
+ scheduler Claimer
+ reader Reader
+ writer Writer
+ factory DeduperFactory
+ contributions wikiContributionStore
ttl time.Duration
heartbeat time.Duration
@@ -84,6 +85,7 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {
reader: engineReader{eng: engine.Get()},
writer: engineWriter{eng: engine.Get()},
factory: defaultDeduperFactory,
+ contributions: newWikiContributionStore(engine.Get()),
ttl: 2 * time.Minute,
heartbeat: 20 * time.Second,
pollInterval: 2 * time.Second,
@@ -168,6 +170,10 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
if paused {
return
}
+ if err := startDatasetCompileLog(ctx, cr.TenantID, datasetID, cr.Token, cr.Entries); err != nil {
+ common.Warn("knowledge_compile: failed to create dataset ingestion log",
+ zap.String("dataset_id", datasetID), zap.Error(err))
+ }
// Heartbeat refreshes the claim TTL while we process; a failed touch means
// the lease was taken over (or reclaimed) and we must abort without acking.
@@ -206,10 +212,18 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
case <-ctx.Done():
close(stopHb)
<-done
+ if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, common.STOPPED, "Wiki dataset compilation stopped during shutdown", -1); err != nil {
+ common.Warn("knowledge_compile: failed to finalize stopped dataset ingestion log",
+ zap.String("dataset_id", datasetID), zap.Error(err))
+ }
return // graceful shutdown: leave inflight for reclamation, do not ack
case <-hbFailed:
close(stopHb)
<-done
+ if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, common.STOPPED, "Wiki dataset compilation lease was lost and will be retried", -1); err != nil {
+ common.Warn("knowledge_compile: failed to finalize stopped dataset ingestion log",
+ zap.String("dataset_id", datasetID), zap.Error(err))
+ }
return // lease lost: do not ack; sweeper/redelivery reprocesses (idempotent)
case <-done:
close(stopHb)
@@ -230,12 +244,44 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {
common.Warn("knowledge_compile: failed to record error_msg",
zap.String("dataset_id", datasetID), zap.Error(err))
}
+ if err := finishDatasetCompileLog(ctx, cr.Token, common.FAILED, batchErr.Error(), -1); err != nil {
+ common.Warn("knowledge_compile: failed to finalize dataset ingestion log",
+ zap.String("dataset_id", datasetID), zap.Error(err))
+ }
return
}
if _, err := c.scheduler.Ack(ctx, datasetID, cr.Token, cr.Entries); err != nil {
common.Warn("knowledge_compile: ack failed",
zap.String("dataset_id", datasetID), zap.Error(err))
+ if logErr := finishDatasetCompileLog(ctx, cr.Token, common.FAILED, "Failed to acknowledge completed Wiki compilation: "+err.Error(), -1); logErr != nil {
+ common.Warn("knowledge_compile: failed to finalize dataset ingestion log",
+ zap.String("dataset_id", datasetID), zap.Error(logErr))
+ }
+ return
+ }
+ if err := finishDatasetCompileLog(ctx, cr.Token, common.COMPLETED, "Wiki dataset compilation completed", 1); err != nil {
+ common.Warn("knowledge_compile: failed to finalize dataset ingestion log",
+ zap.String("dataset_id", datasetID), zap.Error(err))
+ }
+}
+
+func (c *Consumer) reportProgress(ctx context.Context, tenant, datasetID, token string, progress float64, phase, message string) {
+ if err := c.scheduler.UpdateProgress(ctx, datasetID, token, progress, phase, message); err != nil {
+ common.Warn("knowledge_compile: failed to update progress",
+ zap.String("dataset_id", datasetID),
+ zap.String("phase", phase),
+ zap.Error(err))
}
+ if err := updateDatasetCompileLog(ctx, token, progress, message); err != nil {
+ common.Warn("knowledge_compile: failed to update dataset ingestion log",
+ zap.String("dataset_id", datasetID), zap.String("phase", phase), zap.Error(err))
+ }
+ common.Info("knowledge_compile: progress",
+ zap.String("dataset_id", datasetID),
+ zap.String("tenant_id", tenant),
+ zap.Float64("progress", progress),
+ zap.String("phase", phase),
+ zap.String("message", message))
}
var errClaimSuperseded = errors.New("knowledge_compile: claim superseded by rewrite")
@@ -317,7 +363,10 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)
if derr := c.writer.DeleteMergedForVariant(ctx, tenant, kb, nil); derr != nil {
return derr
}
- return c.writer.DropWikiGraph(ctx, tenant, kb)
+ if derr := c.writer.DropWikiGraph(ctx, tenant, kb); derr != nil {
+ return derr
+ }
+ return c.contributions.ClearDataset(ctx, tenant, kb)
}); err != nil {
return fmt.Errorf("knowledge_compile: rebuild clear all variants + graph: %w", err)
}
@@ -444,6 +493,7 @@ func filterWikiPageCandidates(candidates []kccommon.Product) []kccommon.Product
// returns an error if any reader/dedup/writer step fails so the caller can
// leave the batch for reclamation instead of acking dropped work.
func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, entries []BacklogEntry) error {
+ c.reportProgress(ctx, tenant, kb, token, 0.05, "classifying", fmt.Sprintf("Classifying %d scheduling entries", len(entries)))
common.Info("knowledge_compile: processing claimed batch",
zap.String("dataset_id", kb),
zap.String("tenant_id", tenant),
@@ -527,6 +577,33 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
zap.Strings("entries", entryDetails))
if len(deleted) == 0 && len(completed) == 0 {
+ c.reportProgress(ctx, tenant, kb, token, 1, "completed", "No document products require merging")
+ return nil
+ }
+ deletedSet := make(map[string]bool, len(deleted))
+ for _, d := range deleted {
+ deletedSet[d] = true
+ }
+ wikiDiff, err := c.prepareWikiContributionDiff(ctx, tenant, kb, completed, deleted)
+ if err != nil {
+ return err
+ }
+ c.reportProgress(ctx, tenant, kb, token, 0.10, "comparing_wiki_contributions",
+ fmt.Sprintf("Comparing Wiki contributions: %d affected page(s)", len(wikiDiff.affectedKeys)))
+ wikiOnly := len(completed) > 0
+ for _, entry := range completed {
+ if len(entry.Variants) != 1 || !variantsContain(entry.Variants, "wiki") {
+ wikiOnly = false
+ break
+ }
+ }
+ if len(deleted) == 0 && wikiOnly && len(wikiDiff.affectedKeys) == 0 {
+ c.mu.Lock()
+ for _, docID := range pendingTombClear {
+ delete(c.tombs[kb], docID)
+ }
+ c.mu.Unlock()
+ c.reportProgress(ctx, tenant, kb, token, 1, "completed", "Wiki contributions are unchanged; no dataset pages require updating")
return nil
}
@@ -546,11 +623,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
return "deduper factory returned nil"
}()))
}
-
- deletedSet := make(map[string]bool, len(deleted))
- for _, d := range deleted {
- deletedSet[d] = true
- }
+ c.reportProgress(ctx, tenant, kb, token, 0.12, "deleting", fmt.Sprintf("Deleting products for %d document(s)", len(deleted)))
// --- Deletion (two sequential DocEngine calls, no in-memory load) ---
// Deleted wins regardless of batch order, so we process deletions first.
@@ -614,21 +687,39 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
// completed and deleted is a stale tombstone: the deletion wins, so we skip
// its completion.
var incoming []kccommon.Product
+ c.reportProgress(ctx, tenant, kb, token, 0.28, "loading_products", fmt.Sprintf("Loading products for %d document(s)", len(completed)))
for _, e := range completed {
if deletedSet[e.DocID] {
continue
}
- docProducts, err := c.reader.LoadDocProducts(ctx, tenant, kb, e.DocID)
- if err != nil {
- return err
- }
+ docProducts := wikiDiff.productsByDoc[e.DocID]
// A0-4: the event's Variants are the authoritative dispatch signal — route
// ONLY the products whose variant the compiler actually produced for this
// doc. A doc re-compiled under a new template may still hold stale
// doc-level rows of an earlier variant in storage; without this gate those
// would leak into the nav/structure paths despite the event asking only
// for, say, wiki. Legacy events with empty Variants keep all products.
- incoming = append(incoming, productsForVariants(docProducts, e.Variants)...)
+ selected := productsForVariants(docProducts, e.Variants)
+ for _, product := range selected {
+ if product.Variant != kccommon.VariantWiki {
+ incoming = append(incoming, product)
+ }
+ }
+ }
+ if len(wikiDiff.affectedKeys) > 0 {
+ reader, ok := c.reader.(documentWikiPageReader)
+ if !ok {
+ return fmt.Errorf("knowledge_compile: reader does not support Wiki contribution loading")
+ }
+ pages, err := reader.LoadDocumentWikiPagesBySlugs(ctx, tenant, kb, sortedStringSet(wikiDiff.affectedSlugs))
+ if err != nil {
+ return err
+ }
+ for _, page := range pages {
+ if _, affected := wikiDiff.affectedKeys[wikiPageMergeKey(page)]; affected {
+ incoming = append(incoming, page)
+ }
+ }
}
// A0-4 dispatch: route tree/structure products to the dataset-navigation tree
@@ -638,6 +729,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
// so both upsert their summary into the cross-document nav tree; wiki products
// continue to the page-specific evidence merge.
navIn := navInputFromProducts(kb, incoming)
+ c.reportProgress(ctx, tenant, kb, token, 0.40, "merging_navigation", fmt.Sprintf("Merging navigation products: %d", len(navIn)))
if len(navIn) > 0 {
ns := nav.GetNavService()
if ns == nil {
@@ -661,6 +753,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
}
return err
}
+ c.reportProgress(ctx, tenant, kb, token, 0.50, "merging_structure", fmt.Sprintf("Merging structure products: %d", len(incoming)))
// wiki_incremental port (M1): the dataset-level merge only processes wiki
// PAGES. A wiki doc yields both page and section products (Meta.kind
@@ -671,13 +764,31 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
// contract. Legacy rows whose kind is empty are derived in the Reader; only
// truly page-kind wiki products proceed.
candidates := filterWikiPageCandidates(incoming)
+ c.reportProgress(ctx, tenant, kb, token, 0.54, "filtering_candidates", fmt.Sprintf("Filtering Wiki page candidates: %d", len(candidates)))
entityMerged, topicCandidates, err := c.mergeEntityModeCandidates(ctx, tenant, kb, candidates)
if err != nil {
return err
}
+ // A Wiki page is identified by its page type and canonical slug. Do not use
+ // content KNN or an LLM to decide whether two pages are the same: different
+ // slugs are distinct pages, while equal slugs are deterministically folded.
+ topicMerged, err := c.mergeWikiProductsBySlug(ctx, tenant, kb, topicCandidates)
+ if err != nil {
+ return err
+ }
+ entityMerged = append(entityMerged, topicMerged...)
+ if len(wikiDiff.affectedKeys) > 0 {
+ if err := c.deleteMissingScopedWikiPages(ctx, tenant, kb, token, wikiDiff.affectedKeys, entityMerged); err != nil {
+ return err
+ }
+ }
+ c.reportProgress(ctx, tenant, kb, token, 0.58, "merging_by_slug", fmt.Sprintf("Merging Wiki pages by slug: %d", len(entityMerged)))
// Entity pages have stable page identity and must bypass topic-mode KNN and
// Page-specific entity/concept merging. Only topic-mode pages continue below.
- candidates = topicCandidates
+ // The slug-only merge above has already handled every Wiki page candidate.
+ // Leave the legacy generic dedup path with no candidates so it cannot apply
+ // content similarity or LLM merge decisions to Wiki pages.
+ candidates = nil
// In-memory dedup among the completed batch first.
candidates, dedupErr := deduper.Dedup(ctx, candidates)
if dedupErr != nil {
@@ -837,6 +948,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
if err := runCompilerJobs(ctx, jobs); err != nil {
return err
}
+ c.reportProgress(ctx, tenant, kb, token, 0.72, "routing_pages", fmt.Sprintf("Routing %d Wiki pages against existing products", len(candidates)))
// KNN only narrows topic-page candidates. When the hit is a topic page,
// require an explicit topic match or an LLM topic-route decision before
@@ -892,6 +1004,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
var newMerged []kccommon.Product
newMerged = append(newMerged, entityMerged...)
if len(groupsByID) > 0 {
+ c.reportProgress(ctx, tenant, kb, token, 0.82, "llm_merge", fmt.Sprintf("Merging %d candidate groups with the LLM", len(groupsByID)))
// Diagnostics: summarize the KNN groups before the LLM merge decision so
// the reader can see which existing merged rows were hit and by how many
// candidates (e.g. whether a "吕布" candidate hit an existing 吕布 row and
@@ -943,6 +1056,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
mergedFinal = append(mergedFinal, unmatched...)
mergedFinal, staleTopicIDs := mergeTopicProducts(tenant, kb, mergedFinal)
mergedFinal = refreshWikiProductVectors(ctx, tenant, mergedFinal)
+ c.reportProgress(ctx, tenant, kb, token, 0.90, "writing_products", fmt.Sprintf("Writing %d merged products", len(mergedFinal)))
if err := c.withWriteLock(ctx, kb, token, func() error {
if err := c.writer.WriteMerged(ctx, tenant, kb, mergedFinal); err != nil {
return err
@@ -973,6 +1087,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
// then just drops any stale graph rows). The graph is reconstructible, so the
// cost of an occasional no-op reprojection is accepted. It is also a locked
// destructive side effect for the same TOCTOU reasons as WriteMerged.
+ c.reportProgress(ctx, tenant, kb, token, 0.97, "projecting_graph", "Projecting the Wiki graph")
if err := c.withWriteLock(ctx, kb, token, func() error {
return c.writer.ProjectWikiGraph(ctx, tenant, kb)
}); err != nil {
@@ -982,6 +1097,9 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
}
return err
}
+ if err := c.commitWikiContributions(ctx, tenant, kb, wikiDiff.currentByDoc, deleted); err != nil {
+ return err
+ }
// All merge and delete paths succeeded: commit the staged tombstone
// clears. These are only now persisted so a failed batch leaves c.tombs
@@ -996,6 +1114,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
zap.Int("completed_docs", len(completed)),
zap.Int("deleted_docs", len(deleted)),
zap.Int("merged_rows_written", len(mergedFinal)))
+ c.reportProgress(ctx, tenant, kb, token, 1, "completed", fmt.Sprintf("Wiki compilation completed: %d merged products", len(mergedFinal)))
return nil
}
@@ -1007,13 +1126,12 @@ func copyMeta(input map[string]any) map[string]any {
return output
}
-// mergeEntityModeCandidates handles the stage-3 entity page contract. Entity
-// and concept pages are keyed by their deterministic slug, not by embedding
-// proximity. A batch may contain evidence for the same page from multiple
-// documents; fold it before WriteMerged and retain the existing page body when
-// it is already present. Topic pages remain on the KNN/LLM route below.
+// mergeEntityModeCandidates handles entity/concept pages by their deterministic
+// page identity. A batch may contain evidence for the same page from multiple
+// documents; fold it before WriteMerged. Topic pages are returned for the same
+// slug-only path.
func (c *Consumer) mergeEntityModeCandidates(ctx context.Context, tenant, kb string, candidates []kccommon.Product) ([]kccommon.Product, []kccommon.Product, error) {
- groups := make(map[string][]kccommon.Product)
+ var entityCandidates []kccommon.Product
var topic []kccommon.Product
for _, candidate := range candidates {
pageType := strings.ToLower(strings.TrimSpace(metaString(candidate.Meta, "page_type")))
@@ -1021,71 +1139,180 @@ func (c *Consumer) mergeEntityModeCandidates(ctx context.Context, tenant, kb str
topic = append(topic, candidate)
continue
}
- key := datasetLevelID(tenant, kb, candidate)
- groups[key] = append(groups[key], candidate)
+ entityCandidates = append(entityCandidates, candidate)
}
- if len(groups) == 0 {
- return nil, topic, nil
+ merged, err := c.mergeWikiProductsBySlug(ctx, tenant, kb, entityCandidates)
+ return merged, topic, err
+}
+
+// mergeWikiProductsBySlug folds document-level Wiki pages into the dataset
+// level using only page_type + slug. Equal identities are merged; different
+// slugs remain independent pages. A single document page is already a complete
+// Wiki page and passes through unchanged; only groups containing multiple
+// document pages need an LLM rewrite.
+func (c *Consumer) mergeWikiProductsBySlug(ctx context.Context, tenant, kb string, candidates []kccommon.Product) ([]kccommon.Product, error) {
+ if len(candidates) == 0 {
+ return nil, nil
}
- keys := make([]string, 0, len(groups))
- for key := range groups {
- keys = append(keys, key)
+
+ groups := make(map[string][]kccommon.Product)
+ order := make([]string, 0, len(candidates))
+ for _, candidate := range candidates {
+ key := wikiPageMergeKey(candidate)
+ if key == "" {
+ // A malformed page without a slug cannot be safely merged with another
+ // page. Keep it isolated so it remains visible for diagnosis.
+ key = "id\x00" + candidate.ID
+ }
+ if _, exists := groups[key]; !exists {
+ order = append(order, key)
+ }
+ groups[key] = append(groups[key], candidate)
}
- sort.Strings(keys)
- reader, canLoadExisting := c.reader.(mergedProductReader)
- deps, depsErr := kccommon.ResolveDeps(tenant, defaultLLMID, defaultEmbedding)
- merged := make([]kccommon.Product, 0, len(keys))
- for _, key := range keys {
+
+ merged := make([]kccommon.Product, 0, len(order))
+ rewriteIndexes := make([]int, 0, len(order))
+ for _, key := range order {
var current kccommon.Product
- if canLoadExisting {
- var err error
- current, err = reader.LoadMergedProduct(ctx, tenant, kb, key)
- if err != nil {
- return nil, nil, err
- }
- }
+ exists := false
for _, candidate := range groups[key] {
candidate.Merged = true
candidate.DocID = kb
- if current.ID == "" {
+ if !exists || current.ID == "" {
current = candidate
+ exists = true
+ continue
+ }
+ if strings.EqualFold(strings.TrimSpace(metaString(current.Meta, "page_type")), "topic") {
+ current = mergeTopicPage(current, candidate)
} else {
- if depsErr == nil {
- current = c.synthesizeEntityPage(ctx, deps, current, candidate)
- } else {
- current = wikiEntityMerge(current, candidate)
- }
+ current = wikiEntityMerge(current, candidate)
}
}
if current.ID != "" {
current.Merged = true
current.DocID = kb
merged = append(merged, current)
+ if len(groups[key]) > 1 {
+ rewriteIndexes = append(rewriteIndexes, len(merged)-1)
+ }
}
}
- return merged, topic, nil
+ if err := rewriteMergedWikiPages(ctx, tenant, merged, rewriteIndexes); err != nil {
+ return nil, err
+ }
+ return merged, nil
}
-// synthesizeEntityPage updates one entity page from the complete old and new
-// evidence. Unlike topic-mode replacement, the model receives both bodies and
-// is required to preserve every factual block. If the model cannot be resolved
-// (for example in an offline test), the deterministic block union remains safe.
-func (c *Consumer) synthesizeEntityPage(ctx context.Context, deps kccommon.Deps, existing, incoming kccommon.Product) kccommon.Product {
- merged := wikiEntityMerge(existing, incoming)
+const wikiPageRewriteSystemPrompt = `You are the final editor for a knowledge-base Wiki page.
+Rewrite the supplied evidence into one coherent Markdown page.
+Preserve every factual detail from the evidence, including names, dates, numbers,
+relationships, quoted statements, and links. Do not invent facts or remove
+information merely to shorten the page. Organize duplicated or fragmented
+evidence into a clear structure. Return Markdown only, without commentary about
+the editing process.`
+
+// rewriteMergedWikiPages lets the LLM compose the final dataset-level page
+// after deterministic slug grouping has selected the evidence that belongs to
+// that page. The calls are independent and use the shared compiler pool.
+func rewriteMergedWikiPages(ctx context.Context, tenant string, pages []kccommon.Product, indexes []int) error {
+ if len(indexes) == 0 {
+ return nil
+ }
+ deps, err := kccommon.ResolveDeps(tenant, defaultLLMID, defaultEmbedding)
+ if err != nil {
+ return fmt.Errorf("resolve Wiki page rewrite dependencies: %w", err)
+ }
if deps.Chat == nil {
- return merged
+ return fmt.Errorf("Wiki page rewrite chat model is unavailable")
}
- resp, err := deps.Chat.Chat(ctx, kccommon.ChatRequest{
- LLMID: defaultLLMID,
- SystemPrompt: "You are an entity wiki editor. Rewrite the complete page from both evidence versions. Preserve every supported factual statement, do not invent facts, and return only Markdown.",
- UserPrompt: fmt.Sprintf("Entity page: %s\n\nEXISTING PAGE:\n%s\n\nNEW EVIDENCE PAGE:\n%s\n\nWrite the complete merged page. Do not omit facts that appear in either version.",
- metaString(existing.Meta, "title"), existing.Content, incoming.Content),
- })
- if err != nil || resp == nil || strings.TrimSpace(resp.Content) == "" {
- return merged
+ maxTokens := deps.ModelMaxOutput
+ if maxTokens <= 0 {
+ maxTokens = 4096
+ }
+ jobs := make([]CompilerJob, 0, len(indexes))
+ for _, idx := range indexes {
+ idx := idx
+ jobs = append(jobs, func() error {
+ page := pages[idx]
+ prompt := fmt.Sprintf("PAGE TYPE: %s\nTITLE: %s\nTOPIC: %s\nENTITIES: %s\nSUMMARY: %s\n\nEVIDENCE MARKDOWN:\n%s",
+ metaString(page.Meta, "page_type"),
+ metaString(page.Meta, "title"),
+ metaString(page.Meta, "topic"),
+ strings.Join(metaStringSlice(page.Meta, "entity_names"), ", "),
+ metaString(page.Meta, "summary"),
+ page.Content)
+ response, err := deps.Chat.Chat(ctx, kccommon.ChatRequest{
+ LLMID: defaultLLMID,
+ SystemPrompt: wikiPageRewriteSystemPrompt,
+ UserPrompt: prompt,
+ Temperature: floatPtr(0.1),
+ MaxTokens: &maxTokens,
+ })
+ if err != nil {
+ return fmt.Errorf("rewrite Wiki page %q: %w", metaString(page.Meta, "slug"), err)
+ }
+ if response == nil {
+ return fmt.Errorf("rewrite Wiki page %q returned no response", metaString(page.Meta, "slug"))
+ }
+ content := strings.TrimSpace(response.Content)
+ if content == "" {
+ return fmt.Errorf("rewrite Wiki page %q returned empty content", metaString(page.Meta, "slug"))
+ }
+ pages[idx].Content = content
+ return nil
+ })
+ }
+ return SubmitCompilerJobs(ctx, jobs)
+}
+
+func floatPtr(value float64) *float64 { return &value }
+
+func wikiPageMergeKey(product kccommon.Product) string {
+ slug := strings.TrimSpace(metaString(product.Meta, "slug"))
+ if slug == "" {
+ return ""
}
- merged.Content = strings.TrimSpace(resp.Content)
- return merged
+ pageType := strings.ToLower(strings.TrimSpace(metaString(product.Meta, "page_type")))
+ return pageType + "\x00" + strings.ToLower(strings.Join(strings.Fields(slug), " "))
+}
+
+type mergedWikiPageDeleter interface {
+ DeleteMergedWikiPages(ctx context.Context, tenant, kb string, ids []string) error
+}
+
+func (c *Consumer) deleteMissingScopedWikiPages(ctx context.Context, tenant, kb, token string, affected map[string]struct{}, current []kccommon.Product) error {
+ reader, readerOK := c.reader.(mergedWikiPageReader)
+ deleter, writerOK := c.writer.(mergedWikiPageDeleter)
+ if !readerOK || !writerOK {
+ return fmt.Errorf("knowledge_compile: scoped Wiki cleanup is unsupported")
+ }
+ present := make(map[string]struct{}, len(current))
+ for _, page := range current {
+ if key := wikiPageMergeKey(page); key != "" {
+ present[key] = struct{}{}
+ }
+ }
+ existing, err := reader.LoadMergedWikiPages(ctx, tenant, kb)
+ if err != nil {
+ return err
+ }
+ ids := make([]string, 0)
+ for _, page := range existing {
+ key := wikiPageMergeKey(page)
+ if _, scoped := affected[key]; !scoped {
+ continue
+ }
+ if _, remains := present[key]; !remains && page.ID != "" {
+ ids = append(ids, page.ID)
+ }
+ }
+ if len(ids) == 0 {
+ return nil
+ }
+ return c.withWriteLock(ctx, kb, token, func() error {
+ return deleter.DeleteMergedWikiPages(ctx, tenant, kb, ids)
+ })
}
func refreshWikiProductVectors(ctx context.Context, tenant string, products []kccommon.Product) []kccommon.Product {
@@ -1114,6 +1341,24 @@ func productContentsForEmbedding(products []kccommon.Product) []string {
return contents
}
+func variantsContain(variants []string, wanted string) bool {
+ for _, variant := range variants {
+ if variant == wanted {
+ return true
+ }
+ }
+ return false
+}
+
+func sortedStringSet(values map[string]struct{}) []string {
+ result := make([]string, 0, len(values))
+ for value := range values {
+ result = append(result, value)
+ }
+ sort.Strings(result)
+ return result
+}
+
// candidateIdentity returns a compact identity string for a product, preferring
// the wiki slug (full "<page_type>/<slug>") when present, else the id/doc_id,
// so KNN/dedup diagnostics can tie a candidate to the page it belongs to.
diff --git a/internal/ingestion/knowledge_compile/dataset_log.go b/internal/ingestion/knowledge_compile/dataset_log.go
new file mode 100644
index 0000000000..c271273281
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/dataset_log.go
@@ -0,0 +1,122 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package knowledge_compile
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+)
+
+const datasetLogDocumentID = "graph_raptor_x"
+
+func datasetCompileLogID(claimToken string) string {
+ sum := sha256.Sum256([]byte("wiki-dataset-log\x00" + claimToken))
+ return hex.EncodeToString(sum[:16])
+}
+
+func startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimToken string, entries []BacklogEntry) error {
+ if kcDB == nil || claimToken == "" {
+ return nil
+ }
+ now := time.Now()
+ status := "1"
+ message := fmt.Sprintf("Created automatic Wiki dataset task for %d document event(s)", len(entries))
+ entryData := make([]any, 0, len(entries))
+ for _, entry := range entries {
+ entryData = append(entryData, map[string]any{
+ "doc_id": entry.DocID,
+ "event_type": entry.EventType,
+ "variants": entry.Variants,
+ })
+ }
+ log := entity.PipelineOperationLog{
+ ID: datasetCompileLogID(claimToken),
+ DocumentID: datasetLogDocumentID,
+ TenantID: tenantID,
+ KbID: datasetID,
+ ParserID: "wiki",
+ DocumentName: "Wiki",
+ DocumentSuffix: "",
+ DocumentType: "dataset",
+ SourceFrom: "knowledgebase",
+ Progress: 0,
+ ProgressMsg: &message,
+ ProcessBeginAt: &now,
+ DSL: entity.JSONMap{"entries": entryData},
+ TaskType: string(entity.PipelineTaskTypeWiki),
+ OperationStatus: common.RUNNING,
+ Status: &status,
+ }
+ return kcDB.WithContext(ctx).Where("id = ?", log.ID).FirstOrCreate(&log).Error
+}
+
+func updateDatasetCompileLog(ctx context.Context, claimToken string, progress float64, message string) error {
+ if kcDB == nil || claimToken == "" {
+ return nil
+ }
+ logID := datasetCompileLogID(claimToken)
+ var log entity.PipelineOperationLog
+ if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {
+ return err
+ }
+ progressMessage := ""
+ if log.ProgressMsg != nil {
+ progressMessage = *log.ProgressMsg
+ }
+ progressMessage = appendProgressMessage(progressMessage, message)
+ duration := log.ProcessDuration
+ if log.ProcessBeginAt != nil {
+ duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())
+ }
+ return kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{
+ "progress": progress,
+ "progress_msg": progressMessage,
+ "process_duration": duration,
+ "operation_status": common.RUNNING,
+ }).Error
+}
+
+func finishDatasetCompileLog(ctx context.Context, claimToken, operationStatus, message string, progress float64) error {
+ if kcDB == nil || claimToken == "" {
+ return nil
+ }
+ logID := datasetCompileLogID(claimToken)
+ var log entity.PipelineOperationLog
+ if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {
+ return err
+ }
+ progressMessage := ""
+ if log.ProgressMsg != nil {
+ progressMessage = *log.ProgressMsg
+ }
+ progressMessage = appendProgressMessage(progressMessage, message)
+ duration := log.ProcessDuration
+ if log.ProcessBeginAt != nil {
+ duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())
+ }
+ return kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{
+ "progress": progress,
+ "progress_msg": progressMessage,
+ "process_duration": duration,
+ "operation_status": operationStatus,
+ }).Error
+}
diff --git a/internal/ingestion/knowledge_compile/dataset_log_test.go b/internal/ingestion/knowledge_compile/dataset_log_test.go
new file mode 100644
index 0000000000..09ccefaf61
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/dataset_log_test.go
@@ -0,0 +1,50 @@
+package knowledge_compile
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+)
+
+func TestDatasetCompileLogLifecycle(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open("file:dataset-log?mode=memory&cache=shared"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err := db.AutoMigrate(&entity.PipelineOperationLog{}); err != nil {
+ t.Fatalf("migrate pipeline log: %v", err)
+ }
+ oldDB := kcDB
+ kcDB = db
+ t.Cleanup(func() { kcDB = oldDB })
+
+ entries := []BacklogEntry{{DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"}}}
+ if err := startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {
+ t.Fatalf("start dataset log: %v", err)
+ }
+ if err := updateDatasetCompileLog(t.Context(), "claim-1", 0.5, "Comparing Wiki contributions: 2 affected page(s)"); err != nil {
+ t.Fatalf("update dataset log: %v", err)
+ }
+ if err := finishDatasetCompileLog(t.Context(), "claim-1", common.COMPLETED, "Wiki dataset compilation completed", 1); err != nil {
+ t.Fatalf("finish dataset log: %v", err)
+ }
+
+ var log entity.PipelineOperationLog
+ if err := db.Where("id = ?", datasetCompileLogID("claim-1")).First(&log).Error; err != nil {
+ t.Fatalf("load dataset log: %v", err)
+ }
+ if log.DocumentID != datasetLogDocumentID || log.TaskType != string(entity.PipelineTaskTypeWiki) {
+ t.Fatalf("unexpected dataset log identity: %+v", log)
+ }
+ if log.OperationStatus != common.COMPLETED || log.Progress != 1 {
+ t.Fatalf("unexpected final state: status=%s progress=%v", log.OperationStatus, log.Progress)
+ }
+ if log.ProgressMsg == nil || !strings.Contains(*log.ProgressMsg, "2 affected page(s)") || !strings.Contains(*log.ProgressMsg, "completed") {
+ t.Fatalf("progress messages were not persisted: %v", log.ProgressMsg)
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index fb3b6364b3..2cc3c1127f 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -65,6 +65,10 @@ type mergedWikiPageReader interface {
LoadMergedWikiPages(ctx context.Context, tenant, kb string) ([]kccommon.Product, error)
}
+type documentWikiPageReader interface {
+ LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant, kb string, slugs []string) ([]kccommon.Product, error)
+}
+
// engineReader loads the per-document compiled products through the global
// DocEngine (§11.6 step 1, §11.7 incremental re-dedup). It depends on the
// process-wide DocEngine obtained via engine.Get(); the engine abstraction owns
@@ -256,6 +260,46 @@ func (r engineReader) LoadMergedWikiPages(ctx context.Context, tenant, kb string
return pages, nil
}
+func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant, kb string, slugs []string) ([]kccommon.Product, error) {
+ eng := r.eng
+ if eng == nil {
+ eng = engine.Get()
+ }
+ if eng == nil || len(slugs) == 0 {
+ return nil, nil
+ }
+ filter := map[string]interface{}{
+ "available_int": 0,
+ "compile_kwd": compileKwdWikiPage,
+ "slug_kwd": slugs,
+ }
+ var pages []kccommon.Product
+ for offset := 0; ; offset += loadDocProductsLimit {
+ res, err := eng.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)}, KbIDs: []string{kb},
+ Limit: loadDocProductsLimit, Offset: offset,
+ OrderBy: (&types.OrderByExpr{}).Asc("id"),
+ SelectFields: append(append([]string(nil), compiledSelectFields...), wikiSelectFields...),
+ Filter: filter,
+ })
+ if err != nil {
+ return nil, err
+ }
+ if res == nil || len(res.Chunks) == 0 {
+ break
+ }
+ for _, row := range res.Chunks {
+ if product, ok := productFromChunkMap(row, tenant, kccommon.VariantWiki); ok && metaString(product.Meta, "kind") == "page" {
+ pages = append(pages, product)
+ }
+ }
+ if len(res.Chunks) < loadDocProductsLimit {
+ break
+ }
+ }
+ return pages, nil
+}
+
// productFromChunkMap reconstructs a kccommon.Product from a stored compiled
// chunk document. It reads the payload from kc_payload (falling back to
// content_with_weight) and the embedding from the q_<dim>_vec column.
diff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.go
index c2c4f14ad1..1701410994 100644
--- a/internal/ingestion/knowledge_compile/scheduler.go
+++ b/internal/ingestion/knowledge_compile/scheduler.go
@@ -123,6 +123,9 @@ type Claimer interface {
// live, so a stale worker whose lease was reclaimed cannot overwrite the
// status of the worker that took over.
SetError(ctx context.Context, datasetID, token, errMsg string) error
+ // UpdateProgress records the current batch phase and progress while the
+ // claim is live. Updates from a stale worker are ignored by token.
+ UpdateProgress(ctx context.Context, datasetID, token string, progress float64, phase, message string) error
// CancelInflight drops the live claim for a dataset back into the backlog so
// a rewrite can start from a clean index. The token is intentionally NOT
@@ -214,7 +217,7 @@ func (s *mysqlScheduler) Provision(ctx context.Context) error {
// each other's backlog), and the notify is always paired with the append so a
// producer never needs a separate Notify call.
func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {
- return s.publish(ctx, tenantID, datasetID, docID, eventType, variants)
+ return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})
}
// loadRow fetches the dataset scheduling row (no lock). Returns nil (no error)
@@ -231,10 +234,13 @@ func (s *mysqlScheduler) loadRow(ctx context.Context, datasetID string) (*entity
}
func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {
+ return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})
+}
+
+func (s *mysqlScheduler) publishEntry(ctx context.Context, tenantID, datasetID string, entry BacklogEntry) error {
if s.db == nil {
return nil
}
- entry := BacklogEntry{DocID: docID, EventType: eventType, Variants: variants}
// KnowledgeCompileDataset.dataset_id is the PRIMARY KEY, so the SELECT ... FOR
// UPDATE below also takes an InnoDB gap lock for a not-yet-existing dataset;
// concurrent publishers for the same dataset serialize on that lock and the
@@ -277,8 +283,8 @@ func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID
common.Info("knowledge_compile: published backlog entry",
zap.String("dataset_id", datasetID),
zap.String("tenant_id", tenantID),
- zap.String("doc_id", docID),
- zap.String("event_type", eventType))
+ zap.String("doc_id", entry.DocID),
+ zap.String("event_type", entry.EventType))
return s.notify(ctx, datasetID)
}
@@ -350,6 +356,9 @@ func (s *mysqlScheduler) claimRow(ctx context.Context, tx *gorm.DB, datasetID st
row.ClaimExpiresAt = &exp
row.State = DatasetStateRunning
row.ErrorMsg = ""
+ row.Progress = 0
+ row.CurrentPhase = "claiming"
+ row.ProgressMsg = ""
if err := tx.Save(&row).Error; err != nil {
return ClaimResult{}, false, err
}
@@ -461,6 +470,55 @@ func (s *mysqlScheduler) SetError(ctx context.Context, datasetID, token, errMsg
return nil
}
+const progressMessageMaxLength = 3000
+
+func appendProgressMessage(previous, message string) string {
+ if message == "" {
+ return previous
+ }
+ if previous == "" {
+ if len([]rune(message)) > progressMessageMaxLength {
+ return string([]rune(message)[len([]rune(message))-progressMessageMaxLength:])
+ }
+ return message
+ }
+ joined := previous + "\n" + message
+ if len([]rune(joined)) > progressMessageMaxLength {
+ runes := []rune(joined)
+ return string(runes[len(runes)-progressMessageMaxLength:])
+ }
+ return joined
+}
+
+// UpdateProgress records a best-effort phase update for the active claim. The
+// token check prevents a worker whose lease was reclaimed from overwriting the
+// progress of the new owner.
+func (s *mysqlScheduler) UpdateProgress(ctx context.Context, datasetID, token string, progress float64, phase, message string) error {
+ if s.db == nil {
+ return nil
+ }
+ if progress < 0 {
+ progress = 0
+ }
+ if progress > 1 {
+ progress = 1
+ }
+ return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ var row entity.KnowledgeCompileDataset
+ if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).
+ Where("dataset_id = ?", datasetID).First(&row).Error; err != nil {
+ return err
+ }
+ if row.ClaimToken != token {
+ return nil
+ }
+ row.Progress = progress
+ row.CurrentPhase = phase
+ row.ProgressMsg = appendProgressMessage(row.ProgressMsg, message)
+ return tx.Save(&row).Error
+ })
+}
+
// CancelInflight drops the live claim for a dataset back into the backlog so a
// rewrite can start from a clean index. It is a no-op when there is no live
// lease. The dropped batch is re-marked pending and is reprocessed after the
@@ -679,14 +737,17 @@ func removeEntries(inflight, batch []BacklogEntry) []BacklogEntry {
// ---- in-memory scheduler for tests ----
type fakeRow struct {
- tenant string
- backlog []BacklogEntry
- inflight []BacklogEntry
- owner string
- token string
- expires *time.Time
- state string
- errorMsg string
+ tenant string
+ backlog []BacklogEntry
+ inflight []BacklogEntry
+ owner string
+ token string
+ expires *time.Time
+ state string
+ errorMsg string
+ progress float64
+ currentPhase string
+ progressMsg string
// writeMu serializes per-dataset writer side effects and rebuilds, mirroring
// the MySQL scheduler's WithDatasetLock row lock.
writeMu sync.Mutex
@@ -716,10 +777,10 @@ func (f *FakeScheduler) Provision(_ context.Context) error { return nil }
// Publish appends one doc event and pushes a notify (same contract as MySQL).
func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {
- return f.publish(tenantID, datasetID, docID, eventType, variants)
+ return f.publishEntry(tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})
}
-func (f *FakeScheduler) publish(tenantID, datasetID, docID, eventType string, variants []string) error {
+func (f *FakeScheduler) publishEntry(tenantID, datasetID string, entry BacklogEntry) error {
f.mu.Lock()
defer f.mu.Unlock()
r, ok := f.rows[datasetID]
@@ -730,7 +791,7 @@ func (f *FakeScheduler) publish(tenantID, datasetID, docID, eventType string, va
if r.tenant == "" {
r.tenant = tenantID
}
- r.backlog = append(r.backlog, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})
+ r.backlog = append(r.backlog, entry)
// Mirror the MySQL scheduler: surface pending unless a worker is running.
if r.state != DatasetStateRunning {
r.state = DatasetStatePending
@@ -767,6 +828,9 @@ func (f *FakeScheduler) Claim(_ context.Context, datasetID string) (ClaimResult,
r.expires = &exp
r.state = DatasetStateRunning
r.errorMsg = ""
+ r.progress = 0
+ r.currentPhase = "claiming"
+ r.progressMsg = ""
return ClaimResult{DatasetID: datasetID, TenantID: r.tenant, Entries: batch, Token: r.token}, true, nil
}
@@ -816,6 +880,25 @@ func (f *FakeScheduler) SetError(_ context.Context, datasetID, token, errMsg str
return nil
}
+func (f *FakeScheduler) UpdateProgress(_ context.Context, datasetID, token string, progress float64, phase, message string) error {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ r, ok := f.rows[datasetID]
+ if !ok || r.token != token {
+ return nil
+ }
+ if progress < 0 {
+ progress = 0
+ }
+ if progress > 1 {
+ progress = 1
+ }
+ r.progress = progress
+ r.currentPhase = phase
+ r.progressMsg = appendProgressMessage(r.progressMsg, message)
+ return nil
+}
+
// WithDatasetLock executes fn under the per-row write mutex, mirroring the MySQL
// scheduler's row-lock semantics: a rebuild and a writer's destructive side
// effect on the same dataset are mutually exclusive. claimToken is read under
diff --git a/internal/ingestion/knowledge_compile/scheduler_test.go b/internal/ingestion/knowledge_compile/scheduler_test.go
index 6f749cbcae..fe592b1773 100644
--- a/internal/ingestion/knowledge_compile/scheduler_test.go
+++ b/internal/ingestion/knowledge_compile/scheduler_test.go
@@ -56,6 +56,29 @@ func TestFakeSchedulerReclaimsInterruptedClaim(t *testing.T) {
}
}
+func TestFakeSchedulerProgressIsClaimScoped(t *testing.T) {
+ f := NewFakeScheduler()
+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), nil); err != nil {
+ t.Fatalf("publish: %v", err)
+ }
+ claim, ok, err := f.Claim(t.Context(), "kb1")
+ if err != nil || !ok {
+ t.Fatalf("claim: ok=%v err=%v", ok, err)
+ }
+ if err := f.UpdateProgress(t.Context(), "kb1", claim.Token, 0.5, "routing_pages", "Routing pages"); err != nil {
+ t.Fatalf("update progress: %v", err)
+ }
+ if err := f.UpdateProgress(t.Context(), "kb1", "stale-token", 0.9, "wrong", "Should be ignored"); err != nil {
+ t.Fatalf("stale update: %v", err)
+ }
+ f.mu.Lock()
+ row := f.rows["kb1"]
+ f.mu.Unlock()
+ if row.progress != 0.5 || row.currentPhase != "routing_pages" || row.progressMsg != "Routing pages" {
+ t.Fatalf("unexpected progress state: %+v", row)
+ }
+}
+
func TestWithWriteLockRejectsSupersededClaim(t *testing.T) {
f := NewFakeScheduler()
if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}); err != nil {
diff --git a/internal/ingestion/knowledge_compile/service.go b/internal/ingestion/knowledge_compile/service.go
index 6377a38269..00541a9acd 100644
--- a/internal/ingestion/knowledge_compile/service.go
+++ b/internal/ingestion/knowledge_compile/service.go
@@ -22,9 +22,11 @@ import (
"fmt"
"time"
- "gorm.io/gorm"
"ragflow/internal/engine"
+ "ragflow/internal/entity"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+
+ "gorm.io/gorm"
)
// Option configures a Consumer.
@@ -77,6 +79,12 @@ func WithWriter(w Writer) Option { return func(c *Consumer) { c.writer = w } }
// WithDeduperFactory overrides the per-tenant Deduper factory.
func WithDeduperFactory(f DeduperFactory) Option { return func(c *Consumer) { c.factory = f } }
+// withWikiContributionStore overrides the durable document-contribution store
+// in package tests.
+func withWikiContributionStore(store wikiContributionStore) Option {
+ return func(c *Consumer) { c.contributions = store }
+}
+
// defaultLLMID / defaultEmbedding are the model ids used when resolving LLM
// deps for the dataset-level deduper. Set via SetModelConfig (typically from the
// server bootstrap). Empty strings make the factory fall back to the noop
@@ -128,11 +136,16 @@ func Provision(ctx context.Context, mq engine.MessageQueue, db *gorm.DB) error {
// Bound the startup AutoMigrate so a slow/unreachable DB cannot block
// startup indefinitely. The caller's ctx is also honoured (cancelled on
// shutdown).
+ runCtx := ctx
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
if err := s.Provision(ctx); err != nil {
return err
}
+ if err := db.WithContext(ctx).AutoMigrate(&entity.WikiDocumentDirty{}); err != nil {
+ return err
+ }
SetScheduler(s)
+ go runWikiDirtyWorker(runCtx, db, s.holder)
return nil
}
diff --git a/internal/ingestion/knowledge_compile/topic_merge.go b/internal/ingestion/knowledge_compile/topic_merge.go
index e9e0899ef9..c84042d338 100644
--- a/internal/ingestion/knowledge_compile/topic_merge.go
+++ b/internal/ingestion/knowledge_compile/topic_merge.go
@@ -68,10 +68,9 @@ func topicKey(topic string) string {
return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(topic)), " "))
}
-func topicSlug(topic string) string {
- return "topic/" + hashStr(topicKey(topic))
-}
-
+// prepareTopicProduct normalizes topic metadata for legacy routing code. The
+// active merge path does not call it to decide identity; slug remains the
+// caller-provided stable page identity and is never replaced by a hash.
func prepareTopicProduct(product kccommon.Product, topic string) kccommon.Product {
product.Meta = copyMeta(product.Meta)
topic = strings.TrimSpace(topic)
@@ -80,25 +79,19 @@ func prepareTopicProduct(product kccommon.Product, topic string) kccommon.Produc
}
product.Meta["page_type"] = "topic"
product.Meta["topic"] = topic
- product.Meta["slug"] = topicSlug(topic)
- if productTitle(product) == "" {
- product.Meta["title"] = topic
- }
return product
}
-// mergeTopicProducts folds pages with the same canonical topic and preserves
-// the first dataset-level page identity.
+// mergeTopicProducts folds only pages with the same canonical slug and
+// preserves the first dataset-level page identity. Similar topics with
+// different slugs remain separate pages.
func mergeTopicProducts(tenant, kb string, products []kccommon.Product) ([]kccommon.Product, []string) {
groups := make(map[string][]kccommon.Product)
order := make([]string, 0, len(products))
for _, product := range products {
- key := candidateIdentity(product)
- if isTopicPage(product) {
- key = topicKey(productTopic(product))
- if key == "" {
- key = candidateIdentity(product)
- }
+ key := wikiPageMergeKey(product)
+ if key == "" {
+ key = candidateIdentity(product)
}
if _, exists := groups[key]; !exists {
order = append(order, key)
@@ -113,19 +106,11 @@ func mergeTopicProducts(tenant, kb string, products []kccommon.Product) ([]kccom
continue
}
current := items[0]
- if isTopicPage(current) {
- existingSlug := metaString(current.Meta, "slug")
- current = prepareTopicProduct(current, productTopic(current))
- if existingSlug != "" && current.Merged && current.DocID == kb {
- current.Meta["slug"] = existingSlug
- }
- }
for _, item := range items[1:] {
if item.ID != "" && item.ID != current.ID && item.Merged && item.DocID == kb {
stale = append(stale, item.ID)
}
if isTopicPage(current) {
- item = prepareTopicProduct(item, productTopic(current))
current = mergeTopicPage(current, item)
} else {
current = wikiEntityMerge(current, item)
@@ -166,7 +151,7 @@ func mergeTopicPage(existing, incoming kccommon.Product) kccommon.Product {
merged.Meta["page_type"] = "topic"
topic := firstTopicString(metaString(incoming.Meta, "topic"), metaString(existing.Meta, "topic"))
merged.Meta["topic"] = topic
- merged.Meta["title"] = firstTopicString(metaString(incoming.Meta, "title"), metaString(existing.Meta, "title"), topic)
+ merged.Meta["title"] = firstTopicString(metaString(existing.Meta, "title"), metaString(incoming.Meta, "title"), topic)
merged.Meta["entity_names"] = unionStrs(metaStringSlice(existing.Meta, "entity_names"), metaStringSlice(incoming.Meta, "entity_names"))
return merged
}
diff --git a/internal/ingestion/knowledge_compile/topic_merge_test.go b/internal/ingestion/knowledge_compile/topic_merge_test.go
index 92a26d610a..0a6dd86eac 100644
--- a/internal/ingestion/knowledge_compile/topic_merge_test.go
+++ b/internal/ingestion/knowledge_compile/topic_merge_test.go
@@ -20,8 +20,8 @@ func TestMergeTopicProductsPreservesEvidenceAndStableTopicIdentity(t *testing.T)
{
ID: "doc-page-2", DocID: "doc-2", Variant: kccommon.VariantWiki, Content: "# History\n\nsecond",
Meta: map[string]any{
- "kind": "page", "page_type": "topic", "topic": " history ", "title": "History",
- "slug": "topic/history-b", "entity_names": []string{"Bob"},
+ "kind": "page", "page_type": "topic", "topic": " history ", "title": "History (updated)",
+ "slug": "topic/history-a", "entity_names": []string{"Bob"},
"source_doc_ids": []string{"doc-2"}, "source_chunk_ids": []string{"chunk-2"},
},
},
@@ -36,8 +36,11 @@ func TestMergeTopicProductsPreservesEvidenceAndStableTopicIdentity(t *testing.T)
if !strings.Contains(merged[0].Content, "first") || !strings.Contains(merged[0].Content, "second") {
t.Fatalf("merged content lost evidence: %q", merged[0].Content)
}
- if got := metaString(merged[0].Meta, "slug"); got != "topic/"+hashStr("history") {
- t.Fatalf("topic slug = %q, want deterministic topic slug", got)
+ if got := metaString(merged[0].Meta, "slug"); got != "topic/history-a" {
+ t.Fatalf("topic slug = %q, want existing slug", got)
+ }
+ if got := metaString(merged[0].Meta, "title"); got != "History" {
+ t.Fatalf("topic title = %q, want existing title", got)
}
if got := metaStringSlice(merged[0].Meta, "entity_names"); len(got) != 2 {
t.Fatalf("entity names = %v, want both entities", got)
@@ -65,7 +68,7 @@ func TestMergeTopicProductsRemovesSupersededDatasetPage(t *testing.T) {
}},
}
merged, stale := mergeTopicProducts("tenant", "kb", products)
- if len(merged) != 1 || len(stale) != 1 || stale[0] != "new-page" {
- t.Fatalf("merged=%#v stale=%#v, want one survivor and new-page stale", merged, stale)
+ if len(merged) != 2 || len(stale) != 0 {
+ t.Fatalf("merged=%#v stale=%#v, want two separate pages", merged, stale)
}
}
diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.go
new file mode 100644
index 0000000000..000a4a349a
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go
@@ -0,0 +1,386 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package knowledge_compile
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "sort"
+ "strings"
+ "sync"
+
+ "ragflow/internal/engine"
+ enginetypes "ragflow/internal/engine/types"
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+)
+
+const wikiContributionCompileKWD = "wiki_contribution_state"
+
+type wikiContributionPage struct {
+ Key string `json:"key"`
+ Slug string `json:"slug"`
+ PageType string `json:"page_type"`
+ Signature string `json:"signature"`
+}
+
+type wikiDocumentContribution struct {
+ DocumentID string `json:"document_id"`
+ Pages []wikiContributionPage `json:"pages"`
+}
+
+type wikiContributionStore interface {
+ Get(ctx context.Context, tenantID, datasetID, documentID string) (wikiDocumentContribution, bool, error)
+ Put(ctx context.Context, tenantID, datasetID string, contribution wikiDocumentContribution) error
+ Delete(ctx context.Context, tenantID, datasetID, documentID string) error
+ ClearDataset(ctx context.Context, tenantID, datasetID string) error
+}
+
+type engineWikiContributionStore struct {
+ engine engine.DocEngine
+}
+
+func newWikiContributionStore(docEngine engine.DocEngine) wikiContributionStore {
+ if docEngine == nil {
+ return &memoryWikiContributionStore{items: map[string]wikiDocumentContribution{}}
+ }
+ return &engineWikiContributionStore{engine: docEngine}
+}
+
+func (s *engineWikiContributionStore) Get(ctx context.Context, tenantID, datasetID, documentID string) (wikiDocumentContribution, bool, error) {
+ key := wikiContributionStateID(datasetID, documentID)
+ result, err := s.engine.Search(ctx, &enginetypes.SearchRequest{
+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
+ KbIDs: []string{datasetID},
+ Limit: 1,
+ SelectFields: []string{"id", "compile_kwd", "content_with_weight"},
+ Filter: map[string]any{
+ "id": []string{key},
+ "compile_kwd": wikiContributionCompileKWD,
+ "available_int": 0,
+ },
+ })
+ if err != nil {
+ return wikiDocumentContribution{}, false, err
+ }
+ if result == nil || len(result.Chunks) == 0 {
+ return wikiDocumentContribution{}, false, nil
+ }
+ var contribution wikiDocumentContribution
+ if err := json.Unmarshal([]byte(mapStoreString(result.Chunks[0]["content_with_weight"])), &contribution); err != nil {
+ return wikiDocumentContribution{}, false, fmt.Errorf("decode Wiki contribution for document %s: %w", documentID, err)
+ }
+ return contribution, true, nil
+}
+
+func (s *engineWikiContributionStore) Put(ctx context.Context, tenantID, datasetID string, contribution wikiDocumentContribution) error {
+ if contribution.DocumentID == "" {
+ return fmt.Errorf("save Wiki contribution: document_id is required")
+ }
+ payload, err := json.Marshal(contribution)
+ if err != nil {
+ return fmt.Errorf("encode Wiki contribution: %w", err)
+ }
+ row := map[string]any{
+ "id": wikiContributionStateID(datasetID, contribution.DocumentID),
+ "doc_id": "wiki_contribution:" + contribution.DocumentID,
+ "tenant_id": tenantID,
+ "kb_id": datasetID,
+ "compile_kwd": wikiContributionCompileKWD,
+ "scope_kwd": "doc",
+ "source_doc_ids": []string{contribution.DocumentID},
+ "content_with_weight": string(payload),
+ "available_int": 0,
+ }
+ _, err = s.engine.InsertChunks(ctx, []map[string]any{row}, fmt.Sprintf("ragflow_%s", tenantID), datasetID)
+ return err
+}
+
+func (s *engineWikiContributionStore) Delete(ctx context.Context, tenantID, datasetID, documentID string) error {
+ _, err := s.engine.DeleteChunks(ctx, map[string]any{
+ "id": []string{wikiContributionStateID(datasetID, documentID)},
+ "kb_id": datasetID,
+ }, fmt.Sprintf("ragflow_%s", tenantID), datasetID)
+ return err
+}
+
+func (s *engineWikiContributionStore) ClearDataset(ctx context.Context, tenantID, datasetID string) error {
+ _, err := s.engine.DeleteChunks(ctx, map[string]any{
+ "kb_id": datasetID,
+ "compile_kwd": wikiContributionCompileKWD,
+ "available_int": 0,
+ }, fmt.Sprintf("ragflow_%s", tenantID), datasetID)
+ return err
+}
+
+func wikiContributionStateID(datasetID, documentID string) string {
+ sum := sha256.Sum256([]byte(datasetID + "\x00" + documentID + "\x00wiki-contribution"))
+ return hex.EncodeToString(sum[:])
+}
+
+type memoryWikiContributionStore struct {
+ mu sync.Mutex
+ items map[string]wikiDocumentContribution
+}
+
+func (s *memoryWikiContributionStore) Get(_ context.Context, _, datasetID, documentID string) (wikiDocumentContribution, bool, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ item, ok := s.items[datasetID+"\x00"+documentID]
+ return item, ok, nil
+}
+
+func (s *memoryWikiContributionStore) Put(_ context.Context, _, datasetID string, contribution wikiDocumentContribution) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.items[datasetID+"\x00"+contribution.DocumentID] = contribution
+ return nil
+}
+
+func (s *memoryWikiContributionStore) Delete(_ context.Context, _, datasetID, documentID string) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ delete(s.items, datasetID+"\x00"+documentID)
+ return nil
+}
+
+func (s *memoryWikiContributionStore) ClearDataset(_ context.Context, _, datasetID string) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ prefix := datasetID + "\x00"
+ for key := range s.items {
+ if strings.HasPrefix(key, prefix) {
+ delete(s.items, key)
+ }
+ }
+ return nil
+}
+
+func buildWikiDocumentContribution(documentID string, products []kccommon.Product) wikiDocumentContribution {
+ pagesByKey := make(map[string]wikiContributionPage)
+ for _, product := range products {
+ if product.Variant != kccommon.VariantWiki || metaString(product.Meta, "kind") != "page" {
+ continue
+ }
+ key := wikiPageMergeKey(product)
+ if key == "" {
+ continue
+ }
+ pagesByKey[key] = wikiContributionPage{
+ Key: key,
+ Slug: strings.TrimSpace(metaString(product.Meta, "slug")),
+ PageType: strings.ToLower(strings.TrimSpace(metaString(product.Meta, "page_type"))),
+ Signature: wikiContributionSignature(product),
+ }
+ }
+ keys := make([]string, 0, len(pagesByKey))
+ for key := range pagesByKey {
+ keys = append(keys, key)
+ }
+ sort.Strings(keys)
+ pages := make([]wikiContributionPage, 0, len(keys))
+ for _, key := range keys {
+ pages = append(pages, pagesByKey[key])
+ }
+ return wikiDocumentContribution{DocumentID: documentID, Pages: pages}
+}
+
+func diffWikiDocumentContribution(previous, current wikiDocumentContribution) (map[string]struct{}, map[string]struct{}) {
+ previousByKey := make(map[string]wikiContributionPage, len(previous.Pages))
+ for _, page := range previous.Pages {
+ previousByKey[page.Key] = page
+ }
+ currentByKey := make(map[string]wikiContributionPage, len(current.Pages))
+ for _, page := range current.Pages {
+ currentByKey[page.Key] = page
+ }
+ affectedKeys := make(map[string]struct{})
+ affectedSlugs := make(map[string]struct{})
+ for key, page := range currentByKey {
+ old, exists := previousByKey[key]
+ if exists && old.Signature == page.Signature {
+ continue
+ }
+ affectedKeys[key] = struct{}{}
+ if page.Slug != "" {
+ affectedSlugs[page.Slug] = struct{}{}
+ }
+ }
+ for key, page := range previousByKey {
+ if _, exists := currentByKey[key]; exists {
+ continue
+ }
+ affectedKeys[key] = struct{}{}
+ if page.Slug != "" {
+ affectedSlugs[page.Slug] = struct{}{}
+ }
+ }
+ return affectedKeys, affectedSlugs
+}
+
+func wikiContributionSignature(product kccommon.Product) string {
+ normalized := struct {
+ Content string
+ Title string
+ Topic string
+ Summary string
+ EntityNames []string
+ RelatedKBPages []string
+ Outlinks []string
+ SourceChunkIDs []string
+ }{
+ Content: strings.TrimSpace(product.Content),
+ Title: strings.TrimSpace(metaString(product.Meta, "title")),
+ Topic: strings.TrimSpace(metaString(product.Meta, "topic")),
+ Summary: strings.TrimSpace(metaString(product.Meta, "summary")),
+ EntityNames: sortedUniqueCopy(metaStringSlice(product.Meta, "entity_names")),
+ RelatedKBPages: sortedUniqueCopy(metaStringSlice(product.Meta, "related_kb_pages")),
+ Outlinks: sortedUniqueCopy(metaStringSlice(product.Meta, "outlinks")),
+ SourceChunkIDs: sortedUniqueCopy(metaStringSlice(product.Meta, "source_chunk_ids")),
+ }
+ payload, _ := json.Marshal(normalized)
+ sum := sha256.Sum256(payload)
+ return hex.EncodeToString(sum[:])
+}
+
+func sortedUniqueCopy(values []string) []string {
+ copyValues := append([]string(nil), values...)
+ sort.Strings(copyValues)
+ out := copyValues[:0]
+ for _, value := range copyValues {
+ if value == "" || len(out) > 0 && out[len(out)-1] == value {
+ continue
+ }
+ out = append(out, value)
+ }
+ return out
+}
+
+type wikiContributionDiffResult struct {
+ productsByDoc map[string][]kccommon.Product
+ currentByDoc map[string]wikiDocumentContribution
+ affectedKeys map[string]struct{}
+ affectedSlugs map[string]struct{}
+}
+
+func (c *Consumer) prepareWikiContributionDiff(ctx context.Context, tenantID, datasetID string, completed []BacklogEntry, deleted []string) (wikiContributionDiffResult, error) {
+ if c.contributions == nil {
+ c.contributions = newWikiContributionStore(nil)
+ }
+ result := wikiContributionDiffResult{
+ productsByDoc: make(map[string][]kccommon.Product, len(completed)),
+ currentByDoc: make(map[string]wikiDocumentContribution, len(completed)),
+ affectedKeys: map[string]struct{}{},
+ affectedSlugs: map[string]struct{}{},
+ }
+ completedByDoc := make(map[string]BacklogEntry, len(completed))
+ for _, entry := range completed {
+ completedByDoc[entry.DocID] = entry
+ products, err := c.reader.LoadDocProducts(ctx, tenantID, datasetID, entry.DocID)
+ if err != nil {
+ return wikiContributionDiffResult{}, err
+ }
+ result.productsByDoc[entry.DocID] = products
+ wikiProducts := products
+ if len(entry.Variants) > 0 && !variantsContain(entry.Variants, "wiki") {
+ wikiProducts = nil
+ }
+ result.currentByDoc[entry.DocID] = buildWikiDocumentContribution(entry.DocID, wikiProducts)
+ }
+
+ allDocIDs := make([]string, 0, len(completed)+len(deleted))
+ for docID := range completedByDoc {
+ allDocIDs = append(allDocIDs, docID)
+ }
+ allDocIDs = append(allDocIDs, deleted...)
+ sort.Strings(allDocIDs)
+
+ previousByDoc := make(map[string]wikiDocumentContribution, len(allDocIDs))
+ missingPrevious := make(map[string]struct{})
+ for _, docID := range allDocIDs {
+ previous, exists, err := c.contributions.Get(ctx, tenantID, datasetID, docID)
+ if err != nil {
+ return wikiContributionDiffResult{}, err
+ }
+ if exists {
+ previousByDoc[docID] = previous
+ } else {
+ missingPrevious[docID] = struct{}{}
+ }
+ }
+
+ // Upgrade legacy datasets lazily. Before contribution snapshots existed, the
+ // only durable ownership record was source_doc_ids on merged pages. Recover
+ // enough page identity from those rows to retract a deleted/disabled document
+ // correctly on its first post-upgrade event.
+ if len(missingPrevious) > 0 {
+ if reader, ok := c.reader.(mergedWikiPageReader); ok {
+ mergedPages, err := reader.LoadMergedWikiPages(ctx, tenantID, datasetID)
+ if err != nil {
+ return wikiContributionDiffResult{}, err
+ }
+ legacyProducts := make(map[string][]kccommon.Product, len(missingPrevious))
+ for _, page := range mergedPages {
+ for _, docID := range metaStringSlice(page.Meta, "source_doc_ids") {
+ if _, wanted := missingPrevious[docID]; wanted {
+ legacyProducts[docID] = append(legacyProducts[docID], page)
+ }
+ }
+ }
+ for docID, products := range legacyProducts {
+ previousByDoc[docID] = buildWikiDocumentContribution(docID, products)
+ }
+ }
+ }
+
+ for _, docID := range allDocIDs {
+ previous := previousByDoc[docID]
+ current, completedNow := result.currentByDoc[docID]
+ if !completedNow {
+ current = wikiDocumentContribution{DocumentID: docID}
+ }
+ keys, slugs := diffWikiDocumentContribution(previous, current)
+ for key := range keys {
+ result.affectedKeys[key] = struct{}{}
+ }
+ for slug := range slugs {
+ result.affectedSlugs[slug] = struct{}{}
+ }
+ }
+ return result, nil
+}
+
+func (c *Consumer) commitWikiContributions(ctx context.Context, tenantID, datasetID string, current map[string]wikiDocumentContribution, deleted []string) error {
+ for documentID, contribution := range current {
+ if len(contribution.Pages) == 0 {
+ if err := c.contributions.Delete(ctx, tenantID, datasetID, documentID); err != nil {
+ return fmt.Errorf("delete Wiki contribution for document %s: %w", documentID, err)
+ }
+ continue
+ }
+ if err := c.contributions.Put(ctx, tenantID, datasetID, contribution); err != nil {
+ return fmt.Errorf("save Wiki contribution for document %s: %w", documentID, err)
+ }
+ }
+ for _, documentID := range deleted {
+ if err := c.contributions.Delete(ctx, tenantID, datasetID, documentID); err != nil {
+ return fmt.Errorf("delete Wiki contribution for document %s: %w", documentID, err)
+ }
+ }
+ return nil
+}
diff --git a/internal/ingestion/knowledge_compile/wiki_contribution_test.go b/internal/ingestion/knowledge_compile/wiki_contribution_test.go
new file mode 100644
index 0000000000..db4531db14
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/wiki_contribution_test.go
@@ -0,0 +1,92 @@
+package knowledge_compile
+
+import (
+ "context"
+ "testing"
+
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+)
+
+func wikiContributionTestProduct(slug, content string, chunkIDs ...string) kccommon.Product {
+ return kccommon.Product{
+ ID: "doc-page-" + slug,
+ DocID: "doc-1",
+ Variant: kccommon.VariantWiki,
+ Content: content,
+ Meta: map[string]any{
+ "kind": "page",
+ "page_type": "entity",
+ "slug": slug,
+ "title": slug,
+ "source_doc_ids": []string{"doc-1"},
+ "source_chunk_ids": chunkIDs,
+ },
+ }
+}
+
+func TestDiffWikiDocumentContributionIgnoresUnchangedPages(t *testing.T) {
+ product := wikiContributionTestProduct("cao-cao", "same content", "chunk-1")
+ previous := buildWikiDocumentContribution("doc-1", []kccommon.Product{product})
+ current := buildWikiDocumentContribution("doc-1", []kccommon.Product{product})
+ keys, slugs := diffWikiDocumentContribution(previous, current)
+ if len(keys) != 0 || len(slugs) != 0 {
+ t.Fatalf("unchanged contribution produced diff: keys=%v slugs=%v", keys, slugs)
+ }
+}
+
+func TestDiffWikiDocumentContributionIncludesChangedAndRemovedPages(t *testing.T) {
+ previous := buildWikiDocumentContribution("doc-1", []kccommon.Product{
+ wikiContributionTestProduct("cao-cao", "old content", "chunk-1"),
+ wikiContributionTestProduct("liu-bei", "removed content", "chunk-2"),
+ })
+ current := buildWikiDocumentContribution("doc-1", []kccommon.Product{
+ wikiContributionTestProduct("cao-cao", "new content", "chunk-3"),
+ })
+ keys, slugs := diffWikiDocumentContribution(previous, current)
+ for _, key := range []string{"entity\x00cao-cao", "entity\x00liu-bei"} {
+ if _, ok := keys[key]; !ok {
+ t.Fatalf("affected key %q missing from %v", key, keys)
+ }
+ }
+ for _, slug := range []string{"cao-cao", "liu-bei"} {
+ if _, ok := slugs[slug]; !ok {
+ t.Fatalf("affected slug %q missing from %v", slug, slugs)
+ }
+ }
+}
+
+func TestDiffWikiDocumentContributionTreatsDisabledDocumentAsRetraction(t *testing.T) {
+ previous := buildWikiDocumentContribution("doc-1", []kccommon.Product{
+ wikiContributionTestProduct("cao-cao", "content", "chunk-1"),
+ })
+ current := wikiDocumentContribution{DocumentID: "doc-1"}
+ keys, _ := diffWikiDocumentContribution(previous, current)
+ if _, ok := keys["entity\x00cao-cao"]; !ok {
+ t.Fatalf("disabled document did not retract its page: %v", keys)
+ }
+}
+
+func TestProcessBatchSkipsUnchangedWikiOnlyEvent(t *testing.T) {
+ product := wikiContributionTestProduct("cao-cao", "same content", "chunk-1")
+ store := &memoryWikiContributionStore{items: map[string]wikiDocumentContribution{}}
+ if err := store.Put(t.Context(), "tenant-1", "kb-1", buildWikiDocumentContribution("doc-1", []kccommon.Product{product})); err != nil {
+ t.Fatalf("seed contribution: %v", err)
+ }
+ factoryCalled := false
+ consumer := NewConsumer(NewFakeScheduler(),
+ WithReader(&fakeReader{products: []kccommon.Product{product}}),
+ withWikiContributionStore(store),
+ WithDeduperFactory(func(string) (Deduper, error) {
+ factoryCalled = true
+ return nil, nil
+ }),
+ )
+ if err := consumer.processBatch(context.Background(), "tenant-1", "kb-1", "token-1", []BacklogEntry{{
+ DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"},
+ }}); err != nil {
+ t.Fatalf("processBatch failed: %v", err)
+ }
+ if factoryCalled {
+ t.Fatal("unchanged Wiki-only event initialized the dataset merge path")
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/wiki_dirty.go b/internal/ingestion/knowledge_compile/wiki_dirty.go
new file mode 100644
index 0000000000..eb71fb34a9
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/wiki_dirty.go
@@ -0,0 +1,222 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package knowledge_compile
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "sort"
+ "sync"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+
+ "go.uber.org/zap"
+ "gorm.io/gorm"
+ "gorm.io/gorm/clause"
+)
+
+const (
+ wikiDirtyDebounce = 20 * time.Second
+ wikiDirtyLease = 10 * time.Minute
+ wikiDirtyPoll = time.Second
+)
+
+// WikiDirtyRequest describes one debounced document-level Wiki refresh.
+type WikiDirtyRequest struct {
+ TenantID string
+ DatasetID string
+ DocumentID string
+ Revision uint64
+ AffectedChunkIDs []string
+}
+
+// WikiDirtyCompiler regenerates the affected document-level Wiki products.
+type WikiDirtyCompiler func(context.Context, WikiDirtyRequest) error
+
+var wikiDirtyCompilerState struct {
+ sync.RWMutex
+ compiler WikiDirtyCompiler
+}
+
+// SetWikiDirtyCompiler installs the document-level compiler owned by the task
+// composition root.
+func SetWikiDirtyCompiler(compiler WikiDirtyCompiler) {
+ wikiDirtyCompilerState.Lock()
+ wikiDirtyCompilerState.compiler = compiler
+ wikiDirtyCompilerState.Unlock()
+}
+
+// MarkWikiDocumentDirty durably applies a trailing-edge debounce. Repeated
+// mutations merge chunk ids and postpone execution until twenty seconds after
+// the most recent successful mutation.
+func MarkWikiDocumentDirty(ctx context.Context, tenantID, datasetID, documentID string, chunkIDs []string) error {
+ db := kcDB
+ if db == nil {
+ db = dao.DB
+ }
+ if db == nil {
+ return fmt.Errorf("mark Wiki document dirty: database is unavailable")
+ }
+ if tenantID == "" || datasetID == "" || documentID == "" {
+ return fmt.Errorf("mark Wiki document dirty: tenant, dataset, and document ids are required")
+ }
+ runAfter := time.Now().Add(wikiDirtyDebounce)
+ return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ row := entity.WikiDocumentDirty{DocumentID: documentID}
+ err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("document_id = ?", documentID).First(&row).Error
+ if err != nil && err != gorm.ErrRecordNotFound {
+ return err
+ }
+ if err == gorm.ErrRecordNotFound {
+ row = entity.WikiDocumentDirty{
+ DocumentID: documentID,
+ DatasetID: datasetID,
+ TenantID: tenantID,
+ Revision: 1,
+ AffectedChunkIDs: marshalDirtyChunkIDs(chunkIDs),
+ RunAfter: runAfter,
+ State: entity.WikiDirtyStatePending,
+ }
+ return tx.Create(&row).Error
+ }
+ row.TenantID = tenantID
+ row.DatasetID = datasetID
+ row.Revision++
+ row.AffectedChunkIDs = marshalDirtyChunkIDs(append(parseDirtyChunkIDs(row.AffectedChunkIDs), chunkIDs...))
+ row.RunAfter = runAfter
+ row.State = entity.WikiDirtyStatePending
+ row.ErrorMsg = ""
+ row.ClaimOwner = ""
+ row.ClaimExpiresAt = nil
+ return tx.Save(&row).Error
+ })
+}
+
+func runWikiDirtyWorker(ctx context.Context, db *gorm.DB, owner string) {
+ ticker := time.NewTicker(wikiDirtyPoll)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ for {
+ row, ok, err := claimWikiDirty(ctx, db, owner)
+ if err != nil {
+ common.Warn("wiki dirty worker: claim failed", zap.Error(err))
+ break
+ }
+ if !ok {
+ break
+ }
+ processWikiDirty(ctx, db, owner, row)
+ }
+ }
+ }
+}
+
+func claimWikiDirty(ctx context.Context, db *gorm.DB, owner string) (entity.WikiDocumentDirty, bool, error) {
+ var claimed entity.WikiDocumentDirty
+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ now := time.Now()
+ query := tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}).
+ Where("run_after <= ? AND (state = ? OR (state = ? AND claim_expires_at <= ?))",
+ now, entity.WikiDirtyStatePending, entity.WikiDirtyStateRunning, now).
+ Order("run_after ASC").Limit(1)
+ if err := query.First(&claimed).Error; err != nil {
+ return err
+ }
+ expires := now.Add(wikiDirtyLease)
+ return tx.Model(&entity.WikiDocumentDirty{}).Where("document_id = ? AND revision = ?", claimed.DocumentID, claimed.Revision).
+ Updates(map[string]interface{}{
+ "state": entity.WikiDirtyStateRunning,
+ "claim_owner": owner,
+ "claim_expires_at": expires,
+ }).Error
+ })
+ if err == gorm.ErrRecordNotFound {
+ return entity.WikiDocumentDirty{}, false, nil
+ }
+ return claimed, err == nil, err
+}
+
+func processWikiDirty(ctx context.Context, db *gorm.DB, owner string, row entity.WikiDocumentDirty) {
+ wikiDirtyCompilerState.RLock()
+ compiler := wikiDirtyCompilerState.compiler
+ wikiDirtyCompilerState.RUnlock()
+ request := WikiDirtyRequest{
+ TenantID: row.TenantID,
+ DatasetID: row.DatasetID,
+ DocumentID: row.DocumentID,
+ Revision: row.Revision,
+ AffectedChunkIDs: parseDirtyChunkIDs(row.AffectedChunkIDs),
+ }
+ var err error
+ if compiler == nil {
+ err = fmt.Errorf("Wiki dirty compiler is not configured")
+ } else {
+ err = compiler(ctx, request)
+ }
+ if err == nil {
+ db.WithContext(ctx).Where("document_id = ? AND revision = ? AND claim_owner = ?", row.DocumentID, row.Revision, owner).
+ Delete(&entity.WikiDocumentDirty{})
+ return
+ }
+ common.Warn("wiki dirty worker: compile failed",
+ zap.String("document_id", row.DocumentID), zap.Uint64("revision", row.Revision), zap.Error(err))
+ next := time.Now().Add(wikiDirtyDebounce)
+ db.WithContext(ctx).Model(&entity.WikiDocumentDirty{}).
+ Where("document_id = ? AND revision = ? AND claim_owner = ?", row.DocumentID, row.Revision, owner).
+ Updates(map[string]interface{}{
+ "state": entity.WikiDirtyStatePending,
+ "run_after": next,
+ "claim_owner": "",
+ "claim_expires_at": nil,
+ "error_msg": err.Error(),
+ })
+}
+
+func parseDirtyChunkIDs(raw string) []string {
+ var ids []string
+ _ = json.Unmarshal([]byte(raw), &ids)
+ return uniqueSortedStrings(ids)
+}
+
+func marshalDirtyChunkIDs(ids []string) string {
+ payload, _ := json.Marshal(uniqueSortedStrings(ids))
+ return string(payload)
+}
+
+func uniqueSortedStrings(values []string) []string {
+ seen := make(map[string]struct{}, len(values))
+ out := make([]string, 0, len(values))
+ for _, value := range values {
+ if value == "" {
+ continue
+ }
+ if _, ok := seen[value]; ok {
+ continue
+ }
+ seen[value] = struct{}{}
+ out = append(out, value)
+ }
+ sort.Strings(out)
+ return out
+}
diff --git a/internal/ingestion/knowledge_compile/wiki_dirty_test.go b/internal/ingestion/knowledge_compile/wiki_dirty_test.go
new file mode 100644
index 0000000000..04cfe59820
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/wiki_dirty_test.go
@@ -0,0 +1,99 @@
+package knowledge_compile
+
+import (
+ "context"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+
+ "ragflow/internal/entity"
+)
+
+func TestMarkWikiDocumentDirtyDebouncesAndMergesChunks(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err := db.AutoMigrate(&entity.WikiDocumentDirty{}); err != nil {
+ t.Fatalf("migrate dirty table: %v", err)
+ }
+ previousDB := kcDB
+ kcDB = db
+ t.Cleanup(func() { kcDB = previousDB })
+
+ if err := MarkWikiDocumentDirty(t.Context(), "tenant", "dataset", "document", []string{"chunk-b", "chunk-a"}); err != nil {
+ t.Fatalf("first dirty mark: %v", err)
+ }
+ var first entity.WikiDocumentDirty
+ if err := db.First(&first, "document_id = ?", "document").Error; err != nil {
+ t.Fatalf("load first dirty mark: %v", err)
+ }
+ time.Sleep(time.Millisecond)
+ if err := MarkWikiDocumentDirty(t.Context(), "tenant", "dataset", "document", []string{"chunk-c", "chunk-b"}); err != nil {
+ t.Fatalf("second dirty mark: %v", err)
+ }
+ var second entity.WikiDocumentDirty
+ if err := db.First(&second, "document_id = ?", "document").Error; err != nil {
+ t.Fatalf("load second dirty mark: %v", err)
+ }
+
+ if second.Revision != 2 {
+ t.Fatalf("revision = %d, want 2", second.Revision)
+ }
+ if !second.RunAfter.After(first.RunAfter) {
+ t.Fatalf("run_after was not postponed: first=%s second=%s", first.RunAfter, second.RunAfter)
+ }
+ if got, want := parseDirtyChunkIDs(second.AffectedChunkIDs), []string{"chunk-a", "chunk-b", "chunk-c"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("affected chunks = %v, want %v", got, want)
+ }
+ if delay := time.Until(second.RunAfter); delay < wikiDirtyDebounce-time.Second || delay > wikiDirtyDebounce+time.Second {
+ t.Fatalf("debounce delay = %s, want about %s", delay, wikiDirtyDebounce)
+ }
+}
+
+func TestWikiDirtyRevisionPreventsDroppingConcurrentMutation(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err := db.AutoMigrate(&entity.WikiDocumentDirty{}); err != nil {
+ t.Fatalf("migrate dirty table: %v", err)
+ }
+ previousDB := kcDB
+ kcDB = db
+ t.Cleanup(func() { kcDB = previousDB })
+
+ if err := MarkWikiDocumentDirty(t.Context(), "tenant", "dataset", "document", []string{"chunk-a"}); err != nil {
+ t.Fatalf("dirty mark: %v", err)
+ }
+ if err := db.Model(&entity.WikiDocumentDirty{}).Where("document_id = ?", "document").Updates(map[string]any{
+ "state": entity.WikiDirtyStateRunning, "claim_owner": "worker",
+ }).Error; err != nil {
+ t.Fatalf("claim dirty row: %v", err)
+ }
+ var claimed entity.WikiDocumentDirty
+ if err := db.First(&claimed, "document_id = ?", "document").Error; err != nil {
+ t.Fatalf("load claimed row: %v", err)
+ }
+
+ wikiDirtyCompilerState.Lock()
+ previousCompiler := wikiDirtyCompilerState.compiler
+ wikiDirtyCompilerState.compiler = func(_ context.Context, _ WikiDirtyRequest) error {
+ return MarkWikiDocumentDirty(t.Context(), "tenant", "dataset", "document", []string{"chunk-b"})
+ }
+ wikiDirtyCompilerState.Unlock()
+ t.Cleanup(func() { SetWikiDirtyCompiler(previousCompiler) })
+
+ processWikiDirty(t.Context(), db, "worker", claimed)
+
+ var pending entity.WikiDocumentDirty
+ if err := db.First(&pending, "document_id = ?", "document").Error; err != nil {
+ t.Fatalf("concurrent mutation was dropped: %v", err)
+ }
+ if pending.Revision != 2 || pending.State != entity.WikiDirtyStatePending {
+ t.Fatalf("pending row = %+v, want revision 2 pending", pending)
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/wiki_map_store.go b/internal/ingestion/knowledge_compile/wiki_map_store.go
index 88e40985e0..85b32358f1 100644
--- a/internal/ingestion/knowledge_compile/wiki_map_store.go
+++ b/internal/ingestion/knowledge_compile/wiki_map_store.go
@@ -29,6 +29,7 @@ import (
const (
wikiMapExtractCompileKWD = "wiki_map_extract"
+ wikiMapActiveCompileKWD = "wiki_map_active"
wikiMapStoreBatchSize = 500
)
@@ -36,6 +37,53 @@ type wikiMapVersionStore struct {
engine engine.DocEngine
}
+func (s *wikiMapVersionStore) GetWikiMapActiveState(ctx context.Context, tenantID, datasetID, key string) ([]byte, error) {
+ if s == nil || s.engine == nil {
+ return nil, fmt.Errorf("wiki MAP active-state DocStore is not initialized")
+ }
+ result, err := s.engine.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
+ KbIDs: []string{datasetID},
+ Limit: 1,
+ SelectFields: []string{"id", "compile_kwd", "content_with_weight"},
+ Filter: map[string]interface{}{
+ "id": []string{key},
+ "compile_kwd": wikiMapActiveCompileKWD,
+ "available_int": 0,
+ },
+ })
+ if err != nil || result == nil || len(result.Chunks) == 0 {
+ return nil, err
+ }
+ payload := mapStoreString(result.Chunks[0]["content_with_weight"])
+ if payload == "" {
+ return nil, nil
+ }
+ return []byte(payload), nil
+}
+
+func (s *wikiMapVersionStore) PutWikiMapActiveState(ctx context.Context, state kccommon.WikiMapActiveState) error {
+ if s == nil || s.engine == nil {
+ return fmt.Errorf("wiki MAP active-state DocStore is not initialized")
+ }
+ if state.Key == "" || state.TenantID == "" || state.DatasetID == "" || state.DocumentID == "" {
+ return fmt.Errorf("save Wiki MAP active state: key and scope are required")
+ }
+ row := map[string]interface{}{
+ "id": state.Key,
+ "doc_id": "wiki_map_active:" + state.DocumentID,
+ "tenant_id": state.TenantID,
+ "kb_id": state.DatasetID,
+ "compile_kwd": wikiMapActiveCompileKWD,
+ "scope_kwd": "doc",
+ "source_doc_ids": []string{state.DocumentID},
+ "content_with_weight": string(state.Payload),
+ "available_int": 0,
+ }
+ _, err := s.engine.InsertChunks(ctx, []map[string]interface{}{row}, fmt.Sprintf("ragflow_%s", state.TenantID), state.DatasetID)
+ return err
+}
+
// NewWikiMapVersionStore returns the DocStore-backed immutable Wiki MAP cache.
// Its rows remain non-searchable through available_int=0 and the compile
// discriminator, while preserving every chunk/hash version for reuse.
diff --git a/internal/ingestion/knowledge_compile/wiki_merge.go b/internal/ingestion/knowledge_compile/wiki_merge.go
index 14a89f17c9..4a843d21e7 100644
--- a/internal/ingestion/knowledge_compile/wiki_merge.go
+++ b/internal/ingestion/knowledge_compile/wiki_merge.go
@@ -105,7 +105,11 @@ func splitMarkdownBlocks(markdown string) []string {
}
// unionWikiProvenance returns a new Meta map based on a (the existing row). The
-// candidate b overwrites kind / slug / page_type / title / summary /
+// candidate b contributes content metadata, while the existing page identity
+// (slug/title/page_type) remains authoritative because pages are merged only
+// when their canonical slug is equal. This prevents a retry or another
+// document's equivalent page from renaming the dataset-level page.
+// The candidate b overwrites kind / summary /
// entity_names / related_kb_pages / outlinks. Identity and creation time
// (created_at_unix, created_at) stay from a. Source provenance arrays
// (source_doc_ids, source_chunk_ids) are unioned and deduped.
@@ -116,7 +120,7 @@ func unionWikiProvenance(a, b map[string]any) map[string]any {
}
// The incoming page replaces current page metadata. Identity and creation
// time remain from the existing row.
- for _, key := range []string{"slug", "page_type", "title", "summary", "kind"} {
+ for _, key := range []string{"summary", "kind"} {
if v, ok := b[key]; ok {
out[key] = v
}
diff --git a/internal/ingestion/knowledge_compile/wiki_merge_test.go b/internal/ingestion/knowledge_compile/wiki_merge_test.go
index 4c9cccf8f8..9361e69638 100644
--- a/internal/ingestion/knowledge_compile/wiki_merge_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_merge_test.go
@@ -1,118 +1,37 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
package knowledge_compile
import (
- "context"
- "strings"
"testing"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
)
-// TestWikiMergePreservesCandidateVector verifies that page evidence merging
-// retains both Markdown blocks and uses the candidate embedding for the merged
-// body.
-func TestWikiMergePreservesCandidateVector(t *testing.T) {
- existing := kccommon.Product{
- ID: "kb/wiki/alpha",
- DocID: "kb",
- TenantID: "t1",
- Variant: kccommon.VariantWiki,
- Content: "# old markdown",
- Vector: []float32{1, 0, 0, 0},
+func TestMergeWikiProductsBySlugKeepsSingleDocumentPageContent(t *testing.T) {
+ page := kccommon.Product{
+ ID: "doc-page-1",
+ DocID: "doc-1",
+ Variant: kccommon.VariantWiki,
+ Content: "# Original\n\nDocument-level Wiki content.",
Meta: map[string]any{
- "kind": "page",
- "slug": "page/alpha",
- "created_at": "2024-01-01",
+ "kind": "page",
+ "page_type": "entity",
+ "slug": "entity/alpha",
+ "title": "Alpha",
},
}
- candidate := kccommon.Product{
- ID: "kb/wiki/alpha",
- DocID: "kb",
- TenantID: "t1",
- Variant: kccommon.VariantWiki,
- Content: "# new markdown",
- Vector: []float32{0, 1, 0, 0},
- Meta: map[string]any{
- "kind": "page",
- "slug": "page/alpha",
- "run_id": "run-9",
- },
- }
-
- groups := wikiMergeBatch(context.Background(), []MergeGroup{{
- Existing: existing,
- Candidates: []kccommon.Product{candidate},
- }})
-
- merged := groups[0].Merged
- if !strings.Contains(merged.Content, "# old markdown") || !strings.Contains(merged.Content, "# new markdown") {
- t.Fatalf("content = %q, want both evidence blocks", merged.Content)
- }
- if !groups[0].Duplicate {
- t.Fatalf("wiki candidate must be marked duplicate (replacement)")
- }
- if len(merged.Vector) != len(candidate.Vector) {
- t.Fatalf("merged vector must be copied from the candidate, got %v", merged.Vector)
- }
- for i := range candidate.Vector {
- if merged.Vector[i] != candidate.Vector[i] {
- t.Fatalf("merged vector[%d] = %v, want candidate %v", i, merged.Vector, candidate.Vector)
- }
- }
- // Identity and creation time come from the existing row; content-bearing and
- // page metadata fields come from the candidate.
- if merged.ID != existing.ID || merged.DocID != existing.DocID {
- t.Fatalf("identity must be preserved: id=%q doc_id=%q", merged.ID, merged.DocID)
- }
- if merged.Meta["created_at"] != "2024-01-01" {
- t.Fatalf("creation time must be preserved from the existing row, got %v", merged.Meta["created_at"])
- }
-}
-func TestWikiEntityMergeRetainsBothEvidenceBodies(t *testing.T) {
- existing := kccommon.Product{
- ID: "page-1", DocID: "kb", Variant: kccommon.VariantWiki,
- Content: "# Alpha\n\nold evidence",
- Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "page_type": "entity", "source_doc_ids": []string{"doc-1"}},
- }
- incoming := kccommon.Product{
- ID: "doc-page", DocID: "doc-2", Variant: kccommon.VariantWiki,
- Content: "# Alpha\n\nnew evidence",
- Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "page_type": "entity", "source_doc_ids": []string{"doc-2"}},
+ consumer := &Consumer{}
+ merged, err := consumer.mergeWikiProductsBySlug(t.Context(), "tenant-1", "kb-1", []kccommon.Product{page})
+ if err != nil {
+ t.Fatalf("mergeWikiProductsBySlug failed: %v", err)
}
- merged := wikiEntityMerge(existing, incoming)
- if merged.Content == existing.Content || merged.Content == incoming.Content {
- t.Fatalf("entity merge discarded evidence: %q", merged.Content)
+ if len(merged) != 1 {
+ t.Fatalf("merged pages = %d, want 1", len(merged))
}
- if len(metaStringSliceAny(merged.Meta, "source_doc_ids")) != 2 {
- t.Fatalf("source docs = %#v, want both documents", merged.Meta["source_doc_ids"])
- }
- if merged.ID != existing.ID || merged.DocID != existing.DocID {
- t.Fatalf("entity identity changed: %#v", merged)
- }
-}
-
-func TestSplitMarkdownBlocksKeepsFencedCodeTogether(t *testing.T) {
- blocks := splitMarkdownBlocks("before\n\n```go\nline one\n\nline two\n```\n\nafter")
- if len(blocks) != 3 {
- t.Fatalf("blocks=%d, want 3: %#v", len(blocks), blocks)
+ if merged[0].Content != page.Content {
+ t.Fatalf("single-source content changed: got %q, want %q", merged[0].Content, page.Content)
}
- if !strings.Contains(blocks[1], "line one\n\nline two") {
- t.Fatalf("fenced block was split: %q", blocks[1])
+ if merged[0].DocID != "kb-1" || !merged[0].Merged {
+ t.Fatalf("dataset page identity not applied: doc_id=%q merged=%t", merged[0].DocID, merged[0].Merged)
}
}
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index ed2079fb15..caeb784d90 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -1089,7 +1089,7 @@ type wikiPageProjection struct {
// re-materializes the page graph. See the Writer interface doc for the
// delete-then-insert (non-atomic) contract.
func (w engineWriter) ProjectWikiGraph(ctx context.Context, tenant, kb string) error {
- pages, err := w.loadMergedWikiPages(ctx, tenant, kb)
+ pages, err := w.loadActiveDocumentWikiPages(ctx, tenant, kb)
if err != nil {
return err
}
@@ -1116,6 +1116,76 @@ func (w engineWriter) ProjectWikiGraph(ctx context.Context, tenant, kb string) e
return w.insertWikiGraphChunks(ctx, tenant, kb, rows)
}
+// loadActiveDocumentWikiPages loads the immutable document-level page
+// contributions that are currently enabled and folds equal slugs without
+// touching their content. Graph projection must use these rows instead of the
+// LLM-composed dataset page, otherwise disabling one of several contributors
+// would leave that contributor's entities and edges in the graph.
+func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, kb string) ([]wikiPageProjection, error) {
+ eng := w.eng
+ if eng == nil {
+ eng = engine.Get()
+ }
+ if eng == nil {
+ return nil, nil
+ }
+ const batchSize = 2000
+ bySlug := make(map[string]wikiPageProjection)
+ for offset := 0; ; offset += batchSize {
+ result, err := eng.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)},
+ KbIDs: []string{kb},
+ Filter: map[string]interface{}{
+ "compile_kwd": compileKwdWikiPage, "available_int": 0,
+ "scope_kwd": "doc", "kb_id": kb,
+ },
+ SelectFields: []string{
+ "slug_kwd", "page_type_kwd", "title_kwd", "entity_names_kwd",
+ "summary_with_weight", "outlinks_kwd", "source_doc_ids", "source_chunk_ids",
+ },
+ Limit: batchSize, Offset: offset,
+ OrderBy: (&types.OrderByExpr{}).Asc("slug_kwd").Asc("doc_id"),
+ })
+ if err != nil {
+ return nil, err
+ }
+ if result == nil || len(result.Chunks) == 0 {
+ break
+ }
+ for _, row := range result.Chunks {
+ slug := metaString(row, "slug_kwd")
+ if slug == "" {
+ continue
+ }
+ page := bySlug[slug]
+ if page.Slug == "" {
+ page.Slug = slug
+ page.PageType = metaString(row, "page_type_kwd")
+ page.Title = metaString(row, "title_kwd")
+ page.Summary = metaString(row, "summary_with_weight")
+ }
+ page.Aliases = unionStrs(page.Aliases, metaStringSlice(row, "entity_names_kwd"))
+ page.Outlinks = unionStrs(page.Outlinks, metaStringSlice(row, "outlinks_kwd"))
+ page.SourceDocIDs = unionStrs(page.SourceDocIDs, metaStringSlice(row, "source_doc_ids"))
+ page.SourceChunkIDs = unionStrs(page.SourceChunkIDs, metaStringSlice(row, "source_chunk_ids"))
+ bySlug[slug] = page
+ }
+ if len(result.Chunks) < batchSize {
+ break
+ }
+ }
+ slugs := make([]string, 0, len(bySlug))
+ for slug := range bySlug {
+ slugs = append(slugs, slug)
+ }
+ sort.Strings(slugs)
+ pages := make([]wikiPageProjection, 0, len(slugs))
+ for _, slug := range slugs {
+ pages = append(pages, bySlug[slug])
+ }
+ return pages, nil
+}
+
// DropWikiGraph deletes every wiki_entity / wiki_relation row for the dataset.
func (w engineWriter) DropWikiGraph(ctx context.Context, tenant, kb string) error {
return w.dropWikiGraph(ctx, tenant, kb)
diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go
index 8f50a92f69..67353c0ef1 100644
--- a/internal/ingestion/pipeline/pipeline.go
+++ b/internal/ingestion/pipeline/pipeline.go
@@ -467,6 +467,7 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParam
// runs), in which case TrackProgress is a no-op — progress is an
// observability concern, not a data dependency.
runCtx = runtime.WithProgressCallback(runCtx, p.componentProgressCallback(ctx))
+ runCtx = runtime.WithProgressMessageCallback(runCtx, p.componentProgressMessageCallback(ctx))
current := cloneMapOrEmpty(inputs)
@@ -712,3 +713,22 @@ func (p *Pipeline) componentProgressCallback(ctx context.Context) runtime.Progre
})
}
}
+
+type detailedProgressSink interface {
+ OnComponentMessage(ctx context.Context, taskID, documentID, component, message string)
+}
+
+func (p *Pipeline) componentProgressMessageCallback(ctx context.Context) runtime.ProgressMessageCallback {
+ sink, ok := p.sink.(detailedProgressSink)
+ if !ok {
+ return nil
+ }
+ return func(component, message string) {
+ common.Info("component progress detail",
+ zap.String("component", component),
+ zap.String("task_id", p.taskID),
+ zap.String("document_id", p.documentID),
+ zap.String("message", message))
+ sink.OnComponentMessage(ctx, p.taskID, p.documentID, component, message)
+ }
+}
diff --git a/internal/ingestion/service/progress_sink.go b/internal/ingestion/service/progress_sink.go
index efb5469e3a..c7f5a826ec 100644
--- a/internal/ingestion/service/progress_sink.go
+++ b/internal/ingestion/service/progress_sink.go
@@ -114,6 +114,38 @@ func (s *progressSink) OnComponentProgress(ctx context.Context, ev pipeline.Prog
}
}
+// OnComponentMessage appends detailed compiler-stage information without
+// creating a lifecycle row. This keeps component_total and completion percent
+// based on actual canvas components while exposing MAP/REDUCE/PLAN/REFINE
+// counts through the existing document progress log.
+func (s *progressSink) OnComponentMessage(ctx context.Context, taskID, docID, component, message string) {
+ if docID == "" || message == "" {
+ return
+ }
+ s.logMu.Lock()
+ defer s.logMu.Unlock()
+ doc, err := s.docSvc.GetDocumentByID(ctx, docID)
+ if err != nil {
+ common.Error(fmt.Sprintf("progressSink: read document %s for detail failed", docID), err)
+ return
+ }
+ if doc == nil {
+ return
+ }
+ run := ""
+ if doc.Run != nil {
+ run = *doc.Run
+ }
+ log, err := s.accumulateLogLocked(ctx, docID, fmt.Sprintf("%s: %s", component, message))
+ if err != nil {
+ common.Error(fmt.Sprintf("progressSink: append detail for document %s failed", docID), err)
+ return
+ }
+ if err := s.docSvc.UpdateRunProgress(ctx, docID, doc.Progress, run, log); err != nil {
+ common.Error(fmt.Sprintf("progressSink: persist detail for document %s failed", docID), err)
+ }
+}
+
// updateDocumentProgress serializes log accumulation with the corresponding
// document write. Without holding the same lock across both operations, a
// slower database write can store an older snapshot after a newer event has
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index 1c2f66f255..10127128be 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -28,7 +28,7 @@ import (
enginetypes "ragflow/internal/engine/types"
"ragflow/internal/entity"
"ragflow/internal/entity/models"
- _ "ragflow/internal/ingestion/component/knowledge_compiler"
+ knowledgecompiler "ragflow/internal/ingestion/component/knowledge_compiler"
kc "ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/ingestion/knowledge_compile"
"ragflow/internal/service"
@@ -54,6 +54,402 @@ func init() {
kc.SetDepsResolver(newKnowledgeCompilerDepsResolver())
kc.SetGroupResolver(newKnowledgeCompilerGroupResolver())
kc.SetTemplateResolver(newKnowledgeCompilerTemplateResolver())
+ knowledge_compile.SetWikiDirtyCompiler(compileDirtyWikiDocument)
+}
+
+func compileDirtyWikiDocument(ctx context.Context, request knowledge_compile.WikiDirtyRequest) error {
+ doc, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, request.DocumentID)
+ if err != nil {
+ if err == gorm.ErrRecordNotFound {
+ return nil
+ }
+ return fmt.Errorf("Wiki dirty compile: load document: %w", err)
+ }
+ if doc.KbID != request.DatasetID || (doc.Status != nil && *doc.Status == "0") {
+ return replaceDirtyWikiProducts(ctx, request, nil, nil, nil, nil, true)
+ }
+ compilerParams, err := loadWikiCompilerParams(ctx, doc)
+ if err != nil {
+ return err
+ }
+ templateIDs, err := resolveWikiTemplateIDs(ctx, request.TenantID, compilerParams)
+ if err != nil {
+ return err
+ }
+ if len(templateIDs) == 0 {
+ return replaceDirtyWikiProducts(ctx, request, nil, nil, nil, nil, true)
+ }
+ sourceChunks, err := loadActiveSourceChunks(ctx, request)
+ if err != nil {
+ return err
+ }
+ if len(sourceChunks) == 0 {
+ return replaceDirtyWikiProducts(ctx, request, nil, nil, nil, nil, true)
+ }
+ compiled := make([]map[string]any, 0)
+ affectedSlugs := make([]string, 0)
+ removedSlugs := make([]string, 0)
+ activeStates := make([]kc.WikiMapActiveState, 0)
+ for _, templateID := range templateIDs {
+ params := copyStringAnyMap(compilerParams)
+ delete(params, "compilation_template_group_id")
+ delete(params, "compilation_template_group_ids")
+ params["compilation_template_id"] = templateID
+ component, err := knowledgecompiler.NewKnowledgeCompilerComponent("Compiler", params)
+ if err != nil {
+ return err
+ }
+ output, err := component.Invoke(ctx, dao.DB, map[string]any{
+ "chunks": sourceChunks,
+ "tenant_id": request.TenantID,
+ "dataset_id": request.DatasetID,
+ "kb_id": request.DatasetID,
+ "doc_id": request.DocumentID,
+ "wiki_incremental": true,
+ })
+ if err != nil {
+ return fmt.Errorf("Wiki dirty compile document %s: %w", request.DocumentID, err)
+ }
+ compiled = append(compiled, wikiCompiledRows(output)...)
+ affectedSlugs = append(affectedSlugs, stringValues(output["wiki_affected_slugs"])...)
+ removedSlugs = append(removedSlugs, stringValues(output["wiki_removed_slugs"])...)
+ states, err := wikiActiveStates(output)
+ if err != nil {
+ return fmt.Errorf("Wiki dirty compile document %s: %w", request.DocumentID, err)
+ }
+ activeStates = append(activeStates, states...)
+ }
+ if len(affectedSlugs) == 0 && len(removedSlugs) == 0 {
+ return persistDirtyWikiActiveStates(ctx, request, activeStates)
+ }
+ return replaceDirtyWikiProducts(ctx, request, compiled, uniqueStrings(affectedSlugs), uniqueStrings(removedSlugs), activeStates, false)
+}
+
+func loadWikiCompilerParams(ctx context.Context, doc *entity.Document) (map[string]any, error) {
+ if params := findCompilerParams(map[string]any(doc.ParserConfig)); params != nil {
+ return params, nil
+ }
+ if doc.PipelineID == nil || strings.TrimSpace(*doc.PipelineID) == "" {
+ return map[string]any{}, nil
+ }
+ canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, dao.DB, *doc.PipelineID)
+ if err != nil {
+ if err == dao.ErrUserCanvasNotFound {
+ return map[string]any{}, nil
+ }
+ return nil, err
+ }
+ if params := findCompilerParams(map[string]any(canvas.DSL)); params != nil {
+ return params, nil
+ }
+ return map[string]any{}, nil
+}
+
+func findCompilerParams(value any) map[string]any {
+ switch typed := value.(type) {
+ case entity.JSONMap:
+ return findCompilerParams(map[string]any(typed))
+ case map[string]any:
+ if _, hasGroup := typed["compilation_template_group_id"]; hasGroup {
+ return copyStringAnyMap(typed)
+ }
+ if _, hasGroups := typed["compilation_template_group_ids"]; hasGroups {
+ return copyStringAnyMap(typed)
+ }
+ if _, hasTemplate := typed["compilation_template_id"]; hasTemplate {
+ return copyStringAnyMap(typed)
+ }
+ for _, child := range typed {
+ if params := findCompilerParams(child); params != nil {
+ return params
+ }
+ }
+ case []any:
+ for _, child := range typed {
+ if params := findCompilerParams(child); params != nil {
+ return params
+ }
+ }
+ }
+ return nil
+}
+
+func resolveWikiTemplateIDs(ctx context.Context, tenantID string, params map[string]any) ([]string, error) {
+ ids := make([]string, 0)
+ if templateID, ok := params["compilation_template_id"].(string); ok && strings.TrimSpace(templateID) != "" {
+ ids = append(ids, strings.TrimSpace(templateID))
+ }
+ groupIDs := stringValues(params["compilation_template_group_id"])
+ groupIDs = append(groupIDs, stringValues(params["compilation_template_group_ids"])...)
+ if len(groupIDs) > 0 {
+ resolved, err := dao.NewCompilationTemplateDAO().ResolveGroupTemplateIDs(ctx, dao.DB, tenantID, uniqueStrings(groupIDs))
+ if err != nil {
+ return nil, err
+ }
+ ids = append(ids, resolved...)
+ }
+ ids = uniqueStrings(ids)
+ if len(ids) == 0 {
+ return nil, nil
+ }
+ var templates []entity.CompilationTemplate
+ if err := dao.DB.WithContext(ctx).
+ Where("id IN ? AND status = ?", ids, string(entity.StatusValid)).Find(&templates).Error; err != nil {
+ return nil, err
+ }
+ wikiIDs := make([]string, 0, len(templates))
+ for _, template := range templates {
+ kind := template.Kind
+ if configured, ok := template.Config["kind"].(string); ok && strings.TrimSpace(configured) != "" {
+ kind = configured
+ }
+ if strings.EqualFold(strings.TrimSpace(kind), "wiki") || strings.EqualFold(strings.TrimSpace(kind), "wiki_page") {
+ wikiIDs = append(wikiIDs, template.ID)
+ }
+ }
+ return uniqueStrings(wikiIDs), nil
+}
+
+func loadActiveSourceChunks(ctx context.Context, request knowledge_compile.WikiDirtyRequest) ([]map[string]any, error) {
+ docEngine := engine.Get()
+ if docEngine == nil {
+ return nil, fmt.Errorf("Wiki dirty compile: document engine is unavailable")
+ }
+ chunks := make([]map[string]any, 0)
+ for offset := 0; ; offset += 1000 {
+ result, err := docEngine.Search(ctx, &enginetypes.SearchRequest{
+ IndexNames: []string{fmt.Sprintf("ragflow_%s", request.TenantID)},
+ KbIDs: []string{request.DatasetID},
+ Offset: offset,
+ Limit: 1000,
+ SelectFields: []string{
+ "id", "doc_id", "docnm_kwd", "content_with_weight", "available_int", "compile_kwd",
+ },
+ Filter: map[string]any{
+ "doc_id": []string{request.DocumentID},
+ "available_int": 1,
+ },
+ })
+ if err != nil {
+ return nil, err
+ }
+ if result == nil {
+ break
+ }
+ for _, row := range result.Chunks {
+ if strings.TrimSpace(anyString(row["compile_kwd"])) != "" {
+ continue
+ }
+ chunks = append(chunks, row)
+ }
+ if len(result.Chunks) == 0 || int64(offset+len(result.Chunks)) >= result.Total {
+ break
+ }
+ }
+ return chunks, nil
+}
+
+func wikiCompiledRows(output map[string]any) []map[string]any {
+ rows := make([]map[string]any, 0)
+ items, _ := output["chunks"].([]any)
+ for _, item := range items {
+ row, ok := item.(map[string]any)
+ if !ok {
+ continue
+ }
+ compileKWD := strings.TrimSpace(anyString(row["compile_kwd"]))
+ if compileKWD != "wiki_page" && compileKWD != "wiki_section" {
+ continue
+ }
+ row["available_int"] = 0
+ rows = append(rows, row)
+ }
+ return rows
+}
+
+func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.WikiDirtyRequest, rows []map[string]any, affectedSlugs, removedSlugs []string, activeStates []kc.WikiMapActiveState, fullReplace bool) error {
+ var dirty entity.WikiDocumentDirty
+ if err := dao.DB.WithContext(ctx).Where("document_id = ?", request.DocumentID).First(&dirty).Error; err != nil {
+ if err == gorm.ErrRecordNotFound {
+ return nil
+ }
+ return err
+ }
+ if dirty.Revision != request.Revision {
+ return nil
+ }
+ docEngine := engine.Get()
+ if docEngine == nil {
+ return fmt.Errorf("Wiki dirty compile: document engine is unavailable")
+ }
+ indexName := fmt.Sprintf("ragflow_%s", request.TenantID)
+ existing, err := docEngine.Search(ctx, &enginetypes.SearchRequest{
+ IndexNames: []string{indexName},
+ KbIDs: []string{request.DatasetID},
+ Limit: 10000,
+ SelectFields: []string{"id", "compile_kwd", "slug_kwd", "parent_id"},
+ Filter: map[string]any{
+ "doc_id": []string{request.DocumentID},
+ "compile_kwd": []string{"wiki_page", "wiki_section"},
+ "available_int": 0,
+ },
+ })
+ if err != nil {
+ return err
+ }
+ newIDs := make(map[string]struct{}, len(rows))
+ for _, row := range rows {
+ if id := anyString(row["id"]); id != "" {
+ newIDs[id] = struct{}{}
+ }
+ }
+ if len(rows) > 0 {
+ if _, err := docEngine.InsertChunks(ctx, rows, indexName, request.DatasetID); err != nil {
+ return err
+ }
+ }
+ affected := make(map[string]struct{}, len(affectedSlugs)+len(removedSlugs))
+ for _, slug := range append(append([]string(nil), affectedSlugs...), removedSlugs...) {
+ affected[slug] = struct{}{}
+ }
+ affectedPageIDs := make(map[string]struct{})
+ if existing != nil && !fullReplace {
+ for _, row := range existing.Chunks {
+ if anyString(row["compile_kwd"]) != "wiki_page" {
+ continue
+ }
+ if _, ok := affected[anyString(row["slug_kwd"])]; ok {
+ affectedPageIDs[anyString(row["id"])] = struct{}{}
+ }
+ }
+ }
+ if existing != nil && fullReplace {
+ for _, row := range existing.Chunks {
+ if anyString(row["compile_kwd"]) == "wiki_page" {
+ removedSlugs = append(removedSlugs, anyString(row["slug_kwd"]))
+ }
+ }
+ removedSlugs = uniqueStrings(removedSlugs)
+ }
+ staleIDs := make([]string, 0)
+ if existing != nil {
+ for _, row := range existing.Chunks {
+ id := anyString(row["id"])
+ _, pageAffected := affectedPageIDs[id]
+ _, sectionAffected := affectedPageIDs[anyString(row["parent_id"])]
+ inScope := fullReplace || pageAffected || sectionAffected
+ if _, keep := newIDs[id]; id != "" && inScope && !keep {
+ staleIDs = append(staleIDs, id)
+ }
+ }
+ }
+ if len(staleIDs) > 0 {
+ if _, err := docEngine.DeleteChunks(ctx, map[string]any{"id": staleIDs, "kb_id": request.DatasetID}, indexName, request.DatasetID); err != nil {
+ return err
+ }
+ }
+ if fullReplace {
+ if err := clearWikiActiveStates(ctx, docEngine, request); err != nil {
+ return err
+ }
+ }
+ if err := putWikiActiveStates(ctx, docEngine, activeStates); err != nil {
+ return err
+ }
+ return knowledge_compile.PublishCompleted(ctx, request.TenantID, request.DatasetID, request.DocumentID, []string{"wiki"})
+}
+
+func clearWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, request knowledge_compile.WikiDirtyRequest) error {
+ _, err := docEngine.DeleteChunks(ctx, map[string]any{
+ "kb_id": request.DatasetID,
+ "compile_kwd": "wiki_map_active",
+ "available_int": 0,
+ "source_doc_ids": []string{request.DocumentID},
+ }, fmt.Sprintf("ragflow_%s", request.TenantID), request.DatasetID)
+ if err != nil {
+ return fmt.Errorf("clear Wiki active MAP state for document %s: %w", request.DocumentID, err)
+ }
+ return nil
+}
+
+func persistDirtyWikiActiveStates(ctx context.Context, request knowledge_compile.WikiDirtyRequest, states []kc.WikiMapActiveState) error {
+ if len(states) == 0 {
+ return nil
+ }
+ var dirty entity.WikiDocumentDirty
+ if err := dao.DB.WithContext(ctx).Where("document_id = ?", request.DocumentID).First(&dirty).Error; err != nil {
+ if err == gorm.ErrRecordNotFound {
+ return nil
+ }
+ return err
+ }
+ if dirty.Revision != request.Revision {
+ return nil
+ }
+ docEngine := engine.Get()
+ if docEngine == nil {
+ return fmt.Errorf("Wiki dirty compile: document engine is unavailable")
+ }
+ return putWikiActiveStates(ctx, docEngine, states)
+}
+
+func putWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, states []kc.WikiMapActiveState) error {
+ if len(states) == 0 {
+ return nil
+ }
+ store, ok := knowledge_compile.NewWikiMapVersionStore(docEngine).(kc.WikiMapActiveStateStore)
+ if !ok {
+ return fmt.Errorf("Wiki dirty compile: active MAP store is unavailable")
+ }
+ for _, state := range states {
+ if err := store.PutWikiMapActiveState(ctx, state); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func copyStringAnyMap(source map[string]any) map[string]any {
+ result := make(map[string]any, len(source))
+ for key, value := range source {
+ result[key] = value
+ }
+ return result
+}
+
+func stringValues(value any) []string {
+ switch typed := value.(type) {
+ case string:
+ return []string{typed}
+ case []string:
+ return typed
+ case []any:
+ result := make([]string, 0, len(typed))
+ for _, item := range typed {
+ if value, ok := item.(string); ok {
+ result = append(result, value)
+ }
+ }
+ return result
+ }
+ return nil
+}
+
+func uniqueStrings(values []string) []string {
+ seen := make(map[string]struct{}, len(values))
+ result := make([]string, 0, len(values))
+ for _, value := range values {
+ value = strings.TrimSpace(value)
+ if value == "" {
+ continue
+ }
+ if _, ok := seen[value]; ok {
+ continue
+ }
+ seen[value] = struct{}{}
+ result = append(result, value)
+ }
+ return result
}
// newKnowledgeCompilerGroupResolver builds the production GroupResolver backed by
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 40a20cebe5..ad9a480c33 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -28,6 +28,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/engine"
+ enginetypes "ragflow/internal/engine/types"
"ragflow/internal/entity"
"ragflow/internal/ingestion/component"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
@@ -261,9 +262,23 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
// Ordinary source chunks stay available_int=1 (the index default).
markCompiledProductsHidden(chunks)
+ oldCompiledProductIDs, err := s.loadDocumentCompiledProductIDs(ctx)
+ if err != nil {
+ return nil, err
+ }
if err := s.indexWriter.Write(ctx, chunks); err != nil {
return nil, err
}
+ if err := s.reconcileDocumentCompiledProducts(ctx, oldCompiledProductIDs, chunks); err != nil {
+ return nil, err
+ }
+ activeStates, err := wikiActiveStates(pipelineOutput)
+ if err != nil {
+ return nil, err
+ }
+ if err := putWikiActiveStates(ctx, engine.Get(), activeStates); err != nil {
+ return nil, fmt.Errorf("persist Wiki active MAP state: %w", err)
+ }
// All chunks are now persisted. Notify the dataset-level post-processing consumer
// (§11) that this document is complete: its compiled products were written
@@ -407,6 +422,86 @@ func markCompiledProductsHidden(chunks []map[string]any) {
}
}
+// loadDocumentCompiledProductIDs snapshots the previous successful document
+// compiler generation before the new pipeline output is written. Reading first
+// avoids relying on immediate search visibility after a bulk index write.
+func (s *PipelineExecutor) loadDocumentCompiledProductIDs(ctx context.Context) ([]string, error) {
+ docEngine := engine.Get()
+ if docEngine == nil || s == nil || s.taskCtx == nil {
+ return nil, nil
+ }
+ const pageSize = 1000
+ indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)
+ oldIDs := make([]string, 0)
+ for offset := 0; ; offset += pageSize {
+ result, err := docEngine.Search(ctx, &enginetypes.SearchRequest{
+ IndexNames: []string{indexName},
+ KbIDs: []string{s.taskCtx.Doc.KbID},
+ Offset: offset,
+ Limit: pageSize,
+ SelectFields: []string{"id", "compile_kwd"},
+ Filter: map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}},
+ })
+ if err != nil {
+ return nil, fmt.Errorf("load document compiler products: %w", err)
+ }
+ if result == nil || len(result.Chunks) == 0 {
+ break
+ }
+ for _, row := range result.Chunks {
+ if strings.TrimSpace(asCompiledKwd(row)) == "" {
+ continue
+ }
+ if id := strings.TrimSpace(anyString(row["id"])); id != "" {
+ oldIDs = append(oldIDs, id)
+ }
+ }
+ if int64(offset+len(result.Chunks)) >= result.Total {
+ break
+ }
+ }
+ return oldIDs, nil
+}
+
+// reconcileDocumentCompiledProducts advances the document-level compiler
+// generation only after the new pipeline output has been persisted. Reparse
+// preparation keeps the previous compiled rows so a failed run does not erase
+// the last usable document Wiki; this method removes rows absent from the new
+// successful generation immediately before the dataset completion event.
+func (s *PipelineExecutor) reconcileDocumentCompiledProducts(ctx context.Context, oldIDs []string, chunks []map[string]any) error {
+ docEngine := engine.Get()
+ if docEngine == nil || s == nil || s.taskCtx == nil {
+ return nil
+ }
+ newIDs := make(map[string]struct{})
+ for _, chunk := range chunks {
+ if strings.TrimSpace(asCompiledKwd(chunk)) == "" {
+ continue
+ }
+ if id := strings.TrimSpace(anyString(chunk["id"])); id != "" {
+ newIDs[id] = struct{}{}
+ }
+ }
+ staleIDs := make([]string, 0, len(oldIDs))
+ for _, id := range oldIDs {
+ if _, keep := newIDs[id]; !keep {
+ staleIDs = append(staleIDs, id)
+ }
+ }
+ const pageSize = 1000
+ indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)
+ for start := 0; start < len(staleIDs); start += pageSize {
+ end := min(start+pageSize, len(staleIDs))
+ if _, err := docEngine.DeleteChunks(ctx, map[string]any{
+ "id": staleIDs[start:end],
+ "kb_id": s.taskCtx.Doc.KbID,
+ }, indexName, s.taskCtx.Doc.KbID); err != nil {
+ return fmt.Errorf("delete stale document compiler products: %w", err)
+ }
+ }
+ return nil
+}
+
// compiledVariants returns the sorted, de-duplicated set of compile types a
// document's compiled products carry. It reads the authoritative
// `compilation_template_kind_kwd` the KnowledgeCompiler component stamps on each
@@ -475,6 +570,63 @@ func asCompiledKwd(c map[string]any) string {
return ""
}
+func wikiActiveStates(output map[string]any) ([]kccommon.WikiMapActiveState, error) {
+ if output == nil || output["wiki_active_map_states"] == nil {
+ return nil, nil
+ }
+ appendState := func(states []kccommon.WikiMapActiveState, value map[string]any) ([]kccommon.WikiMapActiveState, error) {
+ state := kccommon.WikiMapActiveState{
+ Key: strings.TrimSpace(anyString(value["key"])),
+ TenantID: strings.TrimSpace(anyString(value["tenant_id"])),
+ DatasetID: strings.TrimSpace(anyString(value["dataset_id"])),
+ DocumentID: strings.TrimSpace(anyString(value["document_id"])),
+ }
+ switch payload := value["payload"].(type) {
+ case string:
+ state.Payload = []byte(payload)
+ case []byte:
+ state.Payload = append([]byte(nil), payload...)
+ default:
+ return nil, fmt.Errorf("decode Wiki active MAP state: unsupported payload type %T", value["payload"])
+ }
+ if state.Key == "" || state.TenantID == "" || state.DatasetID == "" || state.DocumentID == "" {
+ return nil, fmt.Errorf("decode Wiki active MAP state: incomplete scope")
+ }
+ return append(states, state), nil
+ }
+
+ switch values := output["wiki_active_map_states"].(type) {
+ case []kccommon.WikiMapActiveState:
+ return append([]kccommon.WikiMapActiveState(nil), values...), nil
+ case []map[string]any:
+ states := make([]kccommon.WikiMapActiveState, 0, len(values))
+ for _, value := range values {
+ var err error
+ states, err = appendState(states, value)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return states, nil
+ case []any:
+ states := make([]kccommon.WikiMapActiveState, 0, len(values))
+ for _, raw := range values {
+ value, ok := raw.(map[string]any)
+ if !ok {
+ return nil, fmt.Errorf("decode Wiki active MAP state: unsupported item type %T", raw)
+ }
+ var err error
+ states, err = appendState(states, value)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return states, nil
+ default:
+ return nil, fmt.Errorf("decode Wiki active MAP states: unsupported type %T", output["wiki_active_map_states"])
+ }
+}
+
func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string) {
var dslMap entity.JSONMap
if err := json.Unmarshal([]byte(dsl), &dslMap); err != nil {
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index 80222d12f6..021d409fa9 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -51,6 +51,20 @@ func TestMarkCompiledProductsHidden(t *testing.T) {
}
}
+func TestWikiActiveStatesDecodeCheckpointValues(t *testing.T) {
+ states, err := wikiActiveStates(map[string]any{
+ "wiki_active_map_states": []any{map[string]any{
+ "key": "state-1", "tenant_id": "tenant-1", "dataset_id": "kb-1", "document_id": "doc-1", "payload": `{"plan":[]}`,
+ }},
+ })
+ if err != nil {
+ t.Fatalf("wikiActiveStates failed: %v", err)
+ }
+ if len(states) != 1 || states[0].Key != "state-1" || string(states[0].Payload) != `{"plan":[]}` {
+ t.Fatalf("decoded states = %#v", states)
+ }
+}
+
func makeTaskCtx() *TaskContext {
return &TaskContext{
IngestionTask: &entity.IngestionTask{
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index 23a34f8f39..07567891cb 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -42,6 +42,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
+ "ragflow/internal/ingestion/knowledge_compile"
"ragflow/internal/service"
"ragflow/internal/service/document"
"ragflow/internal/service/nlp"
@@ -1061,6 +1062,7 @@ func (s *ChunkService) SwitchChunks(ctx context.Context, userID, datasetID, docu
return err
}
}
+ s.markWikiDirty(ctx, targetTenantID, datasetID, documentID, chunkIDs)
return nil
}
@@ -1201,6 +1203,9 @@ func (s *ChunkService) UpdateChunk(ctx context.Context, req *service.UpdateChunk
if err != nil {
return fmt.Errorf("failed to update chunk: %w", err)
}
+ if req.Content != nil || req.Available != nil {
+ s.markWikiDirty(ctx, targetTenantID, req.DatasetID, req.DocumentID, []string{req.ChunkID})
+ }
return nil
}
@@ -1273,6 +1278,7 @@ func (s *ChunkService) RemoveChunks(ctx context.Context, req *service.RemoveChun
if err = s.decrementChunkStats(req.DocID, doc.KbID, 0, deletedCount, 0); err != nil {
return deletedCount, fmt.Errorf("failed to update chunk stats: %w", err)
}
+ s.markWikiDirty(ctx, targetTenantID, doc.KbID, req.DocID, req.ChunkIDs)
}
return deletedCount, nil
@@ -1408,6 +1414,7 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkReques
if err = s.incrementChunkStats(req.DocumentID, req.DatasetID, tokenNum, 1, 0); err != nil {
return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("increment chunk stats: %v", err)}
}
+ s.markWikiDirty(ctx, kb.TenantID, req.DatasetID, req.DocumentID, []string{chunkID})
renamedChunk := map[string]interface{}{
"id": chunkID,
@@ -1430,6 +1437,15 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkReques
return &service.AddChunkResponse{Chunk: renamedChunk}, nil
}
+func (s *ChunkService) markWikiDirty(ctx context.Context, tenantID, datasetID, documentID string, chunkIDs []string) {
+ markCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
+ defer cancel()
+ if err := knowledge_compile.MarkWikiDocumentDirty(markCtx, tenantID, datasetID, documentID, chunkIDs); err != nil {
+ common.Warn("chunk mutation: failed to schedule Wiki refresh",
+ zap.String("document_id", documentID), zap.Error(err))
+ }
+}
+
type addChunkError struct {
code common.ErrorCode
message string
diff --git a/internal/service/dataset/compilation_status.go b/internal/service/dataset/compilation_status.go
index 0c996d565b..712ab4cc1d 100644
--- a/internal/service/dataset/compilation_status.go
+++ b/internal/service/dataset/compilation_status.go
@@ -25,6 +25,9 @@ import (
type CompilationStatus struct {
State string `json:"state"` // idle | pending | running | completed
Error string `json:"error,omitempty"` // most recent batch diagnostic (empty when none)
+ Progress float64 `json:"progress"` // current claimed batch progress
+ CurrentPhase string `json:"current_phase,omitempty"` // current compiler phase
+ ProgressMsg string `json:"progress_msg,omitempty"` // bounded phase log for the UI
Inflight int `json:"inflight"` // entries currently claimed (in-flight batch)
Backlog int `json:"backlog"` // entries still waiting to be claimed
LastCompletedAt *time.Time `json:"last_completed_at,omitempty"` // last backlog drain
@@ -61,6 +64,9 @@ func (d *DatasetService) GetDatasetCompilationStatus(ctx context.Context, userID
st.State = entity.DatasetStateIdle
}
st.Error = row.ErrorMsg
+ st.Progress = row.Progress
+ st.CurrentPhase = row.CurrentPhase
+ st.ProgressMsg = row.ProgressMsg
st.Inflight = jsonArrayLen(row.InflightDocIDs)
st.Backlog = jsonArrayLen(row.BacklogDocIDs)
st.LastCompletedAt = row.LastCompletedAt
diff --git a/internal/service/dataset/compilation_status_test.go b/internal/service/dataset/compilation_status_test.go
index 9b910fbc4b..8753061e48 100644
--- a/internal/service/dataset/compilation_status_test.go
+++ b/internal/service/dataset/compilation_status_test.go
@@ -129,6 +129,9 @@ func TestGetDatasetCompilationStatus_FullOutput(t *testing.T) {
InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","seq":1},{"doc_id":"d2","event_type":"completed","seq":2}]`,
State: entity.DatasetStateRunning,
ErrorMsg: "merge failed: boom",
+ Progress: 0.72,
+ CurrentPhase: "routing_pages",
+ ProgressMsg: "Routing 55 Wiki pages",
LastCompletedAt: &lastDone,
}
if err := db.Create(&row).Error; err != nil {
@@ -152,6 +155,9 @@ func TestGetDatasetCompilationStatus_FullOutput(t *testing.T) {
if st.Error != "merge failed: boom" {
t.Fatalf("error=%q want %q", st.Error, "merge failed: boom")
}
+ if st.Progress != 0.72 || st.CurrentPhase != "routing_pages" || st.ProgressMsg != "Routing 55 Wiki pages" {
+ t.Fatalf("progress fields not mapped: progress=%v phase=%q msg=%q", st.Progress, st.CurrentPhase, st.ProgressMsg)
+ }
if st.LastCompletedAt == nil || !st.LastCompletedAt.Equal(lastDone) {
t.Fatalf("last_completed_at=%v want %v", st.LastCompletedAt, lastDone)
}
diff --git a/internal/service/dataset/ingestion.go b/internal/service/dataset/ingestion.go
index 84750a8c47..75ee64fdaf 100644
--- a/internal/service/dataset/ingestion.go
+++ b/internal/service/dataset/ingestion.go
@@ -123,8 +123,15 @@ func datasetIngestionLogToMap(log *entity.PipelineOperationLog) map[string]inter
"task_type": log.TaskType,
"operation_status": log.OperationStatus,
"progress": log.Progress,
+ "process_begin_at": log.ProcessBeginAt,
+ "process_duration": log.ProcessDuration,
+ "progress_msg": log.ProgressMsg,
+ "dsl": log.DSL,
+ "avatar": log.Avatar,
"create_time": log.CreateTime,
+ "create_date": log.CreateDate,
"update_time": log.UpdateTime,
+ "update_date": log.UpdateDate,
}
if log.PipelineID != nil {
m["pipeline_id"] = *log.PipelineID
diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go
index 37ad7cd953..d374bf18e9 100644
--- a/internal/service/dataset_artifact_service.go
+++ b/internal/service/dataset_artifact_service.go
@@ -23,8 +23,11 @@ import (
"golang.org/x/text/collate"
"golang.org/x/text/language"
+ "gorm.io/gorm"
+ "ragflow/internal/dao"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
+ "ragflow/internal/entity"
"ragflow/internal/service/nav"
)
@@ -480,33 +483,206 @@ type WikiAlteration struct {
// GetWikiAlteration returns the wiki alteration summary for a dataset.
func (s *DatasetArtifactService) GetWikiAlteration(ctx context.Context, tenantID, datasetID string) (*WikiAlteration, error) {
- chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}},
- []string{"source_doc_ids"}, 0, 10000, nil)
- if err != nil {
- return nil, err
- }
involved := map[string]struct{}{}
- for _, c := range chunks {
- for _, d := range toStringSlice(c["source_doc_ids"]) {
- involved[d] = struct{}{}
+ for offset := 0; ; offset += 1000 {
+ chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID,
+ map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}},
+ []string{"source_doc_ids"}, offset, 1000, nil)
+ if err != nil {
+ return nil, err
+ }
+ for _, c := range chunks {
+ for _, d := range toStringSlice(c["source_doc_ids"]) {
+ if d != "" {
+ involved[d] = struct{}{}
+ }
+ }
+ }
+ if len(chunks) == 0 || int64(offset+len(chunks)) >= total {
+ break
}
}
- ids := make([]string, 0, len(involved))
- for d := range involved {
- ids = append(ids, d)
+ // The database is the source of truth for the current document set. The
+ // previous implementation returned the source_doc_ids from the compiled
+ // pages as both sides of the comparison, which made deletion impossible to
+ // observe. In particular, a deleted document remains in source_doc_ids until
+ // the dataset-level consumer removes the old page.
+ var documents []entity.Document
+ if err := dao.DB.WithContext(ctx).Where("kb_id = ?", datasetID).Find(&documents).Error; err != nil {
+ return nil, fmt.Errorf("list dataset documents for wiki alteration: %w", err)
+ }
+ eligible := make(map[string]struct{}, len(documents))
+ for i := range documents {
+ if documents[i].Status != nil && *documents[i].Status == "0" {
+ continue
+ }
+ pipelineID := ""
+ if documents[i].PipelineID != nil {
+ pipelineID = *documents[i].PipelineID
+ }
+ ok, err := s.documentHasWikiTemplate(ctx, tenantID, documents[i].ParserConfig, pipelineID)
+ if err != nil {
+ return nil, err
+ }
+ if ok {
+ eligible[documents[i].ID] = struct{}{}
+ }
}
- sort.Strings(ids)
+
+ removedIDs := setDifference(involved, eligible)
+ newlyUploadedIDs := setDifference(eligible, involved)
return &WikiAlteration{
- Removed: 0,
- NewlyUploaded: 0,
- RemovedDocIDs: []string{},
- NewlyUploadedDocIDs: []string{},
- InvolvedDocIDs: ids,
- EligibleDocIDs: ids,
+ Removed: len(removedIDs),
+ NewlyUploaded: len(newlyUploadedIDs),
+ RemovedDocIDs: removedIDs,
+ NewlyUploadedDocIDs: newlyUploadedIDs,
+ InvolvedDocIDs: sortedSetKeys(involved),
+ EligibleDocIDs: sortedSetKeys(eligible),
}, nil
}
+// documentHasWikiTemplate reports whether a document's saved pipeline config
+// contains a valid wiki compiler template. A document is eligible only when it
+// is configured for wiki compilation; treating every document in the dataset
+// as eligible would hide both template removal and document deletion.
+func (s *DatasetArtifactService) documentHasWikiTemplate(ctx context.Context, tenantID string, config entity.JSONMap, pipelineID string) (bool, error) {
+ ok, err := s.valueHasWikiTemplate(ctx, tenantID, config)
+ if err != nil || ok {
+ return ok, err
+ }
+ if pipelineID == "" {
+ return false, nil
+ }
+ var canvas entity.UserCanvas
+ if err := dao.DB.WithContext(ctx).Where("id = ?", pipelineID).First(&canvas).Error; err != nil {
+ if err == gorm.ErrRecordNotFound {
+ return false, nil
+ }
+ return false, fmt.Errorf("load pipeline %q for wiki alteration: %w", pipelineID, err)
+ }
+ return s.valueHasWikiTemplate(ctx, tenantID, canvas.DSL)
+}
+
+// valueHasWikiTemplate recursively inspects persisted parser/pipeline JSON.
+// Pipeline DSLs and document parser configs use different nesting layouts, so
+// looking only at a single top-level Compiler key misses pipeline documents.
+func (s *DatasetArtifactService) valueHasWikiTemplate(ctx context.Context, tenantID string, value interface{}) (bool, error) {
+ if items, ok := value.([]interface{}); ok {
+ for _, item := range items {
+ if found, err := s.valueHasWikiTemplate(ctx, tenantID, item); err != nil || found {
+ return found, err
+ }
+ }
+ return false, nil
+ }
+ params, isMap := value.(map[string]interface{})
+ if !isMap {
+ if typed, ok := value.(entity.JSONMap); ok {
+ params = map[string]interface{}(typed)
+ isMap = true
+ }
+ }
+ if !isMap {
+ return false, nil
+ }
+ if id, ok := params["compilation_template_id"].(string); ok && id != "" {
+ var template entity.CompilationTemplate
+ if err := dao.DB.WithContext(ctx).
+ Where("id = ? AND (tenant_id = ? OR tenant_id IS NULL OR tenant_id = '') AND status = ?", id, tenantID, string(entity.StatusValid)).
+ First(&template).Error; err != nil {
+ if err == gorm.ErrRecordNotFound {
+ return false, nil
+ }
+ return false, fmt.Errorf("load wiki compilation template %q: %w", id, err)
+ }
+ if templateKind(template) == "wiki" {
+ return true, nil
+ }
+ }
+ if groupID, ok := params["compilation_template_group_id"].(string); ok && groupID != "" {
+ if found, err := s.groupHasWikiTemplate(ctx, tenantID, groupID); err != nil || found {
+ return found, err
+ }
+ }
+ if rawGroupIDs, ok := params["compilation_template_group_ids"]; ok {
+ groupIDs := []string{}
+ switch values := rawGroupIDs.(type) {
+ case string:
+ groupIDs = append(groupIDs, values)
+ case []interface{}:
+ for _, value := range values {
+ if groupID, ok := value.(string); ok {
+ groupIDs = append(groupIDs, groupID)
+ }
+ }
+ case []string:
+ groupIDs = values
+ }
+ for _, groupID := range groupIDs {
+ if strings.TrimSpace(groupID) != "" {
+ if found, err := s.groupHasWikiTemplate(ctx, tenantID, groupID); err != nil || found {
+ return found, err
+ }
+ }
+ }
+ }
+ for _, child := range params {
+ if found, err := s.valueHasWikiTemplate(ctx, tenantID, child); err != nil || found {
+ return found, err
+ }
+ }
+ return false, nil
+}
+
+func (s *DatasetArtifactService) groupHasWikiTemplate(ctx context.Context, tenantID, groupID string) (bool, error) {
+ var group entity.CompilationTemplateGroup
+ if err := dao.DB.WithContext(ctx).
+ Where("id = ? AND (tenant_id = ? OR tenant_id = '') AND status = ?", groupID, tenantID, string(entity.StatusValid)).
+ First(&group).Error; err != nil {
+ if err == gorm.ErrRecordNotFound {
+ return false, nil
+ }
+ return false, fmt.Errorf("load compilation template group %q: %w", groupID, err)
+ }
+ var templates []entity.CompilationTemplate
+ if err := dao.DB.WithContext(ctx).Where("group_id = ? AND status = ?", groupID, string(entity.StatusValid)).Find(&templates).Error; err != nil {
+ return false, fmt.Errorf("load compilation template group %q: %w", groupID, err)
+ }
+ for _, template := range templates {
+ if templateKind(template) == "wiki" {
+ return true, nil
+ }
+ }
+ return false, nil
+}
+
+func templateKind(template entity.CompilationTemplate) string {
+ if kind, ok := template.Config["kind"].(string); ok && strings.TrimSpace(kind) != "" {
+ return strings.ToLower(strings.TrimSpace(kind))
+ }
+ return strings.ToLower(strings.TrimSpace(template.Kind))
+}
+
+func setDifference(left, right map[string]struct{}) []string {
+ ids := make([]string, 0)
+ for id := range left {
+ if _, ok := right[id]; !ok {
+ ids = append(ids, id)
+ }
+ }
+ sort.Strings(ids)
+ return ids
+}
+
+func sortedSetKeys(set map[string]struct{}) []string {
+ ids := make([]string, 0, len(set))
+ for id := range set {
+ ids = append(ids, id)
+ }
+ sort.Strings(ids)
+ return ids
+}
+
// ClearWiki deletes all wiki artifacts for a dataset.
func (s *DatasetArtifactService) ClearWiki(ctx context.Context, tenantID, datasetID string) (map[string]int, error) {
docEngine := engine.Get()
@@ -792,6 +968,10 @@ func toStringSlice(v interface{}) []string {
if t == "" {
return []string{}
}
+ var decoded interface{}
+ if json.Unmarshal([]byte(t), &decoded) == nil {
+ return toStringSlice(decoded)
+ }
return []string{t}
case []interface{}:
out := make([]string, 0, len(t))
diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go
index 23edc266f5..17bdb9097a 100644
--- a/internal/service/document/document_crud.go
+++ b/internal/service/document/document_crud.go
@@ -334,7 +334,18 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri
}
common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: failed to delete tasks for %s: %v", docID, delErr))
}
- return s.deleteDocRecordWithCounters(ctx, doc, kb.ID)
+ if err := s.deleteDocRecordWithCounters(ctx, doc, kb.ID); err != nil {
+ return err
+ }
+ // File replacement/deletion uses this path instead of deleteDocumentFull.
+ // Publish the same deletion event so the dataset-level Wiki consumer removes
+ // the deleted document's contribution in both paths.
+ pubCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
+ defer cancel()
+ if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID); err != nil {
+ common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: publish doc_deleted for %s failed: %v", docID, err))
+ }
+ return nil
}
// InsertDocument creates a document row and increments the owning KB's doc_num
diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.go
index d846994cfd..a5f19008f5 100644
--- a/internal/service/document/document_dataset_update.go
+++ b/internal/service/document/document_dataset_update.go
@@ -80,13 +80,7 @@ func (s *DocumentService) BatchUpdateDocumentStatus(ctx context.Context, userID,
hasError = true
continue
}
- err = s.docEngine.UpdateChunks(
- ctx,
- map[string]interface{}{"doc_id": docID},
- map[string]interface{}{"available_int": statusInt},
- fmt.Sprintf("ragflow_%s", kb.TenantID),
- doc.KbID,
- )
+ err = s.updateSourceChunkAvailability(ctx, kb.TenantID, doc.KbID, docID, statusInt)
if err != nil {
_ = s.documentDAO.UpdateByID(ctx, dao.DB, docID, map[string]interface{}{"status": previousStatus})
msg := err.Error()
@@ -99,6 +93,7 @@ func (s *DocumentService) BatchUpdateDocumentStatus(ctx context.Context, userID,
continue
}
}
+ s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, docID)
result[docID] = map[string]string{"status": status}
}
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index b385dfbf72..9ba0823d06 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -111,7 +111,7 @@ func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *en
if !exists {
return nil
}
- if _, err = s.docEngine.DeleteChunks(ctx, map[string]interface{}{"doc_id": doc.ID}, indexName, doc.KbID); err != nil {
+ if err = s.deleteSourceChunks(ctx, tenantID, doc.KbID, doc.ID); err != nil {
return err
}
return nil
@@ -405,7 +405,7 @@ func (s *DocumentService) resetDocumentForReparse(ctx context.Context, doc *enti
if s.docEngine != nil {
indexName := fmt.Sprintf("ragflow_%s", tenantID)
s.deleteChunkImages(ctx, doc, indexName)
- if _, err = s.docEngine.DeleteChunks(ctx, map[string]interface{}{"doc_id": doc.ID}, indexName, doc.KbID); err != nil {
+ if err = s.deleteSourceChunks(ctx, tenantID, doc.KbID, doc.ID); err != nil {
return err
}
}
@@ -517,16 +517,11 @@ func (s *DocumentService) updateDocumentStatusOnly(ctx context.Context, doc *ent
return errors.New("database error (Document update)")
}
- if s.docEngine == nil {
- return nil
+ if s.docEngine != nil {
+ if err := s.updateSourceChunkAvailability(ctx, kb.TenantID, doc.KbID, doc.ID, status); err != nil {
+ return err
+ }
}
-
- indexName := fmt.Sprintf("ragflow_%s", kb.TenantID)
- return s.docEngine.UpdateChunks(
- ctx,
- map[string]interface{}{"doc_id": doc.ID},
- map[string]interface{}{"available_int": status},
- indexName,
- doc.KbID,
- )
+ s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, doc.ID)
+ return nil
}
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index 67e1c53e14..bc161f2b9a 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -26,6 +26,7 @@ import (
"net/http"
"net/http/httptest"
"path/filepath"
+ "reflect"
"strings"
"testing"
"time"
@@ -227,6 +228,13 @@ func (e *rerunDeleteDocEngine) ChunkStoreExists(context.Context, string, string)
return true, nil
}
+func (e *rerunDeleteDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {
+ return &types.SearchResult{Chunks: []map[string]interface{}{
+ {"id": "source-1"},
+ {"id": "wiki-1", "compile_kwd": "wiki_page"},
+ }, Total: 2}, nil
+}
+
func (e *rerunDeleteDocEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {
e.deleteCalls++
e.condition = condition
@@ -235,6 +243,24 @@ func (e *rerunDeleteDocEngine) DeleteChunks(_ context.Context, condition map[str
return 3, nil
}
+type sourceAvailabilityDocEngine struct {
+ fakeChatDocEngine
+ updateConditions []map[string]interface{}
+}
+
+func (e *sourceAvailabilityDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {
+ return &types.SearchResult{Chunks: []map[string]interface{}{
+ {"id": "source-1"},
+ {"id": "wiki-1", "compile_kwd": "wiki_page"},
+ {"id": "map-1", "compile_kwd": []interface{}{"wiki_map_active"}},
+ }, Total: 3}, nil
+}
+
+func (e *sourceAvailabilityDocEngine) UpdateChunks(_ context.Context, condition map[string]interface{}, _ map[string]interface{}, _, _ string) error {
+ e.updateConditions = append(e.updateConditions, condition)
+ return nil
+}
+
type metadataDocEngine struct {
fakeChatDocEngine
records map[string]map[string]interface{}
@@ -1951,11 +1977,28 @@ func TestClearDocumentParseResultsClearsCountersTasksAndChunks(t *testing.T) {
if engine.deleteCalls != 1 {
t.Fatalf("deleteCalls = %d, want 1", engine.deleteCalls)
}
- if engine.indexName != "ragflow_tenant-1" || engine.datasetID != "kb-1" || engine.condition["doc_id"] != "doc-1" {
+ if engine.indexName != "ragflow_tenant-1" || engine.datasetID != "kb-1" || !reflect.DeepEqual(engine.condition["id"], []string{"source-1"}) {
t.Fatalf("unexpected delete call: index=%s dataset=%s condition=%v", engine.indexName, engine.datasetID, engine.condition)
}
}
+func TestUpdateSourceChunkAvailabilityExcludesCompiledProducts(t *testing.T) {
+ docEngine := &sourceAvailabilityDocEngine{}
+ svc := testDocumentService(t)
+ svc.docEngine = docEngine
+
+ if err := svc.updateSourceChunkAvailability(t.Context(), "tenant-1", "kb-1", "doc-1", 1); err != nil {
+ t.Fatalf("updateSourceChunkAvailability failed: %v", err)
+ }
+ if len(docEngine.updateConditions) != 1 {
+ t.Fatalf("UpdateChunks calls = %d, want 1", len(docEngine.updateConditions))
+ }
+ ids, ok := docEngine.updateConditions[0]["id"].([]string)
+ if !ok || len(ids) != 1 || ids[0] != "source-1" {
+ t.Fatalf("updated ids = %#v, want only source-1", docEngine.updateConditions[0]["id"])
+ }
+}
+
func TestClearDocumentParseResultsIsIdempotentForStaleDocSnapshot(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
diff --git a/internal/service/document/wiki_refresh.go b/internal/service/document/wiki_refresh.go
new file mode 100644
index 0000000000..1e27a034fa
--- /dev/null
+++ b/internal/service/document/wiki_refresh.go
@@ -0,0 +1,137 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "time"
+
+ "ragflow/internal/common"
+ enginetypes "ragflow/internal/engine/types"
+ "ragflow/internal/ingestion/knowledge_compile"
+
+ "go.uber.org/zap"
+)
+
+const sourceChunkAvailabilityBatchSize = 1000
+
+// updateSourceChunkAvailability changes retrieval visibility for source chunks
+// only. Document-level compiler products must remain hidden (available_int=0)
+// until the dataset-level compiler has merged them.
+func (s *DocumentService) updateSourceChunkAvailability(ctx context.Context, tenantID, datasetID, documentID string, available int) error {
+ if s.docEngine == nil {
+ return fmt.Errorf("document engine not initialized")
+ }
+ indexName := fmt.Sprintf("ragflow_%s", tenantID)
+ ids, err := s.loadSourceChunkIDs(ctx, indexName, datasetID, documentID)
+ if err != nil {
+ return err
+ }
+ for start := 0; start < len(ids); start += sourceChunkAvailabilityBatchSize {
+ end := min(start+sourceChunkAvailabilityBatchSize, len(ids))
+ if err := s.docEngine.UpdateChunks(ctx,
+ map[string]any{"id": ids[start:end]},
+ map[string]any{"available_int": available},
+ indexName,
+ datasetID,
+ ); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, datasetID, documentID string) ([]string, error) {
+ ids := make([]string, 0)
+ for offset := 0; ; offset += sourceChunkAvailabilityBatchSize {
+ result, err := s.docEngine.Search(ctx, &enginetypes.SearchRequest{
+ IndexNames: []string{indexName},
+ KbIDs: []string{datasetID},
+ Offset: offset,
+ Limit: sourceChunkAvailabilityBatchSize,
+ SelectFields: []string{"id", "compile_kwd"},
+ Filter: map[string]any{"doc_id": []string{documentID}},
+ })
+ if err != nil {
+ return nil, err
+ }
+ if result == nil || len(result.Chunks) == 0 {
+ break
+ }
+ for _, row := range result.Chunks {
+ if strings.TrimSpace(documentStoreString(row["compile_kwd"])) != "" {
+ continue
+ }
+ if id := strings.TrimSpace(documentStoreString(row["id"])); id != "" {
+ ids = append(ids, id)
+ }
+ }
+ if int64(offset+len(result.Chunks)) >= result.Total {
+ break
+ }
+ }
+ return ids, nil
+}
+
+func (s *DocumentService) deleteSourceChunks(ctx context.Context, tenantID, datasetID, documentID string) error {
+ if s.docEngine == nil {
+ return nil
+ }
+ indexName := fmt.Sprintf("ragflow_%s", tenantID)
+ ids, err := s.loadSourceChunkIDs(ctx, indexName, datasetID, documentID)
+ if err != nil || len(ids) == 0 {
+ return err
+ }
+ for start := 0; start < len(ids); start += sourceChunkAvailabilityBatchSize {
+ end := min(start+sourceChunkAvailabilityBatchSize, len(ids))
+ if _, err := s.docEngine.DeleteChunks(ctx, map[string]any{
+ "id": ids[start:end],
+ "kb_id": datasetID,
+ }, indexName, datasetID); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (s *DocumentService) markDocumentWikiDirty(ctx context.Context, tenantID, datasetID, documentID string) {
+ markCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
+ defer cancel()
+ if err := knowledge_compile.MarkWikiDocumentDirty(markCtx, tenantID, datasetID, documentID, nil); err != nil {
+ common.Warn("document mutation: failed to schedule Wiki refresh",
+ zap.String("document_id", documentID), zap.Error(err))
+ }
+}
+
+func documentStoreString(value any) string {
+ switch typed := value.(type) {
+ case string:
+ return typed
+ case []string:
+ if len(typed) == 1 {
+ return typed[0]
+ }
+ case []any:
+ if len(typed) == 1 {
+ if text, ok := typed[0].(string); ok {
+ return text
+ }
+ }
+ }
+ return ""
+}
diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.py
index 3a71d98fbc..81c0b6d087 100644
--- a/rag/advanced_rag/knowlege_compile/wiki.py
+++ b/rag/advanced_rag/knowlege_compile/wiki.py
@@ -844,7 +844,7 @@ async def _wiki_scan_current_chunk_state(
while True:
res = await thread_pool_exec(
settings.docStoreConn.search,
- ["id", "doc_id", "content_with_weight"],
+ ["id", "doc_id", "content_with_weight", "compile_kwd"],
[],
{
"doc_id": [doc_id],
@@ -858,8 +858,18 @@ async def _wiki_scan_current_chunk_state(
index,
[kb_id],
)
- rows = settings.docStoreConn.get_fields(res, ["id", "doc_id", "content_with_weight"]) or {}
+ rows = settings.docStoreConn.get_fields(res, ["id", "doc_id", "content_with_weight", "compile_kwd"]) or {}
for row_id, row in rows.items():
+ # The source-chunk query also has a backend-level
+ # ``must_not exists compile_kwd`` guard. Keep this explicit
+ # check because some doc-store implementations do not apply
+ # that nested filter consistently. Derived Wiki/structure
+ # rows must never become MAP inputs or chunk deltas.
+ compile_kwd = row.get("compile_kwd")
+ if isinstance(compile_kwd, (list, tuple, set)):
+ compile_kwd = next((str(value).strip() for value in compile_kwd if value), "")
+ if str(compile_kwd or "").strip():
+ continue
chunk_id = str(row.get("id") or row_id or "")
if chunk_id:
state[chunk_id] = {
@@ -912,6 +922,39 @@ async def _wiki_load_active_map_state(
if len(rows) < 1000:
break
offset += 1000
+ # Active snapshots are normally source-only because they are committed
+ # from _wiki_scan_current_chunk_state. Older snapshots, however, may have
+ # been created before derived rows were excluded. If a historical state ID
+ # still exists in the index and carries compile_kwd, it is a compiled
+ # product rather than an original chunk and must not be counted as a
+ # deleted source chunk. Missing rows are intentionally retained: they are
+ # the legitimate deleted/removed source chunks we need to report.
+ chunk_ids = list(state)
+ for start in range(0, len(chunk_ids), 500):
+ batch_ids = chunk_ids[start : start + 500]
+ try:
+ res = await thread_pool_exec(
+ settings.docStoreConn.search,
+ ["id", "compile_kwd"],
+ [],
+ {"id": batch_ids},
+ [],
+ OrderByExpr(),
+ 0,
+ len(batch_ids),
+ index,
+ [kb_id],
+ )
+ rows = settings.docStoreConn.get_fields(res, ["id", "compile_kwd"]) or {}
+ except Exception:
+ logging.exception("wiki_map: failed to validate historical source snapshot")
+ continue
+ for row_id, row in rows.items():
+ compile_kwd = row.get("compile_kwd")
+ if isinstance(compile_kwd, (list, tuple, set)):
+ compile_kwd = next((str(value).strip() for value in compile_kwd if value), "")
+ if str(compile_kwd or "").strip():
+ state.pop(str(row.get("id") or row_id), None)
return state
diff --git a/web/src/hooks/use-dataset-generate.ts b/web/src/hooks/use-dataset-generate.ts
index cfb168c9f6..86d3165a6d 100644
--- a/web/src/hooks/use-dataset-generate.ts
+++ b/web/src/hooks/use-dataset-generate.ts
@@ -72,6 +72,7 @@ export interface ITraceInfo {
// are the MySQL scheduling-entry counts, and error is the batch diagnostic
// (NOT a peer state). Only populated by the Go/hybrid branch of useTraceQuery.
compilationState?: string;
+ currentPhase?: string;
inflight?: number;
backlog?: number;
compilationError?: string;
@@ -126,9 +127,10 @@ const useTraceQuery = (
const state: string = st.state ?? 'idle';
const error: string = st.error ?? '';
return {
- progress: state === 'completed' ? 1 : state === 'idle' ? 0 : 0,
- progress_msg: error || state,
+ progress: state === 'completed' ? 1 : (st.progress ?? 0),
+ progress_msg: st.progress_msg || error || state,
compilationState: state,
+ currentPhase: st.current_phase ?? '',
inflight: st.inflight ?? 0,
backlog: st.backlog ?? 0,
compilationError: error,
@@ -283,11 +285,9 @@ export function useGenerateStatus(data?: ITraceInfo) {
const percent = useMemo(() => {
if (isGoDatasetBackend()) {
- // No stable terminal state and no DocTotal/DocProcessed, so there is no
- // meaningful percentage. Failures render as a full error marker; active
- // runs show the inflight/backlog counts instead of a percent (see the
- // UpdateRunProgress / EmptyState components).
- return status === GenerateStatus.Failed ? 100 : 0;
+ return status === GenerateStatus.Failed
+ ? 100
+ : (data?.progress ?? 0) * 100;
}
if (status === GenerateStatus.Failed) {
return 100;
diff --git a/web/src/pages/dataset/compilation/update-button.tsx b/web/src/pages/dataset/compilation/update-button.tsx
index 4ab9ed6f9c..b064a6c241 100644
--- a/web/src/pages/dataset/compilation/update-button.tsx
+++ b/web/src/pages/dataset/compilation/update-button.tsx
@@ -11,7 +11,6 @@ import {
} from '@/components/ui/tooltip';
import { GenerateStatus, GenerateType } from '@/constants/knowledge';
import { ITraceInfo, useGenerateStatus } from '@/hooks/use-dataset-generate';
-import { isGoDatasetBackend } from '@/utils/api-proxy-scheme';
import { UpdateRunProgress } from './update-run-progress';
@@ -43,13 +42,6 @@ export function CompilationUpdateButton({
const isRunning = status === GenerateStatus.Running;
const isGenerating = isRunning || status === GenerateStatus.Failed;
- // Go/hybrid: compilation is auto-driven by the scheduler with no manual
- // re-merge, so hide the update control rather than offer a trigger that
- // cannot work (plan v4.1 §4.2).
- if (isGoDatasetBackend()) {
- return null;
- }
-
// A failed trace persists (progress stays < 0) until the next run, so it
// must not keep the button visible on its own — only real changes or a
// live run should.
diff --git a/web/src/pages/dataset/compilation/update-run-progress.tsx b/web/src/pages/dataset/compilation/update-run-progress.tsx
index f778d1b57d..c5b2dd18ff 100644
--- a/web/src/pages/dataset/compilation/update-run-progress.tsx
+++ b/web/src/pages/dataset/compilation/update-run-progress.tsx
@@ -46,7 +46,8 @@ export function UpdateRunProgress({
<span className="size-2 rounded-full bg-accent-primary" />
{data?.compilationError
? data.compilationError
- : t('knowledgeCompilation.compiling', {
+ : data?.currentPhase ||
+ t('knowledgeCompilation.compiling', {
defaultValue: 'Compiling…',
})}
{!data?.compilationError && (