RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat: improve incremental Wiki compilation (#18557)
buua436a3df5889513e
diff --git a/internal/agent/runtime/helpers.go b/internal/agent/runtime/helpers.goindex c66c1f16ea..31e3b35b2d 100644--- a/internal/agent/runtime/helpers.go+++ b/internal/agent/runtime/helpers.go@@ -88,6 +88,29 @@ type ProgressEvent struct {// ProgressCallback receives progress notifications from TrackProgress.type ProgressCallback func(event ProgressEvent)+// ProgressMessageCallback receives detailed messages emitted by a component+// while it is running. Unlike ProgressCallback, these messages do not change+// the component lifecycle counters; they are supplemental observability for+// stages such as a compiler's MAP/REDUCE/PLAN/REFINE pipeline.+type ProgressMessageCallback func(component, message string)++type progressMessageCallbackKey struct{}++// WithProgressMessageCallback attaches a detailed component-message sink to a+// run context. A nil callback is valid and keeps components DB-independent.+func WithProgressMessageCallback(ctx context.Context, cb ProgressMessageCallback) context.Context {+ return context.WithValue(ctx, progressMessageCallbackKey{}, cb)+}++// ReportProgressMessage forwards a detailed component message when a sink is+// attached. Components can call this without depending on the persistence+// layer or changing the lifecycle progress contract.+func ReportProgressMessage(ctx context.Context, component, message string) {+ if cb, ok := ctx.Value(progressMessageCallbackKey{}).(ProgressMessageCallback); ok && cb != nil {+ cb(component, message)+ }+}+// TrackProgress wraps fn with progress notifications. The callback is// invoked at most twice per call (once at start, once at end)://diff --git a/internal/dao/database.go b/internal/dao/database.goindex 818b0097d4..03f4efda1c 100644--- a/internal/dao/database.go+++ b/internal/dao/database.go@@ -158,6 +158,7 @@ func InitDB(ctx context.Context, migrateDB bool) error {&entity.FileCommit{},&entity.FileCommitItem{},&entity.KnowledgeCompileDataset{},+ &entity.WikiDocumentDirty{},// Knowledge-compile compilation templates and their groups. The Go// KnowledgeCompilerComponent resolves a compilation_template (or group)// from these tables at runtime, so the Go side must guarantee they exist.diff --git a/internal/entity/dataset.go b/internal/entity/dataset.goindex 1c1bad0ddf..101662aeb0 100644--- a/internal/entity/dataset.go+++ b/internal/entity/dataset.go@@ -82,6 +82,7 @@ const (PipelineTaskTypeGraphRAG PipelineTaskType = "GraphRAG"PipelineTaskTypeMindmap PipelineTaskType = "Mindmap"PipelineTaskTypeMemory PipelineTaskType = "Memory"+ PipelineTaskTypeWiki PipelineTaskType = "Wiki")// FileSource represents the source of a filediff --git a/internal/entity/knowledge_compile_doc.go b/internal/entity/knowledge_compile_doc.goindex 1c38685919..099c25d579 100644--- a/internal/entity/knowledge_compile_doc.go+++ b/internal/entity/knowledge_compile_doc.go@@ -61,7 +61,10 @@ type KnowledgeCompileDataset struct {// ErrorMsg is the most recent failure/retry diagnostic. It is TEXT with no// DDL default (MySQL 8.0.13+ rejects a literal default on TEXT, Error 1101);// the application always writes it explicitly when set.- ErrorMsg string `gorm:"column:error_msg;type:text;not null" json:"error_msg"`+ ErrorMsg string `gorm:"column:error_msg;type:text;not null" json:"error_msg"`+ Progress float64 `gorm:"column:progress;not null;default:0" json:"progress"`+ CurrentPhase string `gorm:"column:current_phase;size:64;not null;default:''" json:"current_phase"`+ ProgressMsg string `gorm:"column:progress_msg;type:text;not null" json:"progress_msg"`// LastCompletedAt records the last time the backlog drained to empty.LastCompletedAt *time.Time `gorm:"column:last_completed_at;default:null" json:"last_completed_at"`CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"created_at"`diff --git a/internal/entity/wiki_document_dirty.go b/internal/entity/wiki_document_dirty.gonew file mode 100644index 0000000000..eff002de59--- /dev/null+++ b/internal/entity/wiki_document_dirty.go@@ -0,0 +1,43 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package entity++import "time"++const (+ WikiDirtyStatePending = "pending"+ WikiDirtyStateRunning = "running"+)++// WikiDocumentDirty is the durable trailing-edge debounce row for direct chunk+// mutations. Each document owns one row; another mutation increments Revision,+// merges the affected chunk ids, and moves RunAfter twenty seconds forward.+type WikiDocumentDirty struct {+ DocumentID string `gorm:"primaryKey;column:document_id;size:64" json:"document_id"`+ DatasetID string `gorm:"column:dataset_id;size:64;not null;index" json:"dataset_id"`+ TenantID string `gorm:"column:tenant_id;size:64;not null;index" json:"tenant_id"`+ Revision uint64 `gorm:"column:revision;not null;default:1" json:"revision"`+ AffectedChunkIDs string `gorm:"column:affected_chunk_ids;type:text;not null" json:"affected_chunk_ids"`+ RunAfter time.Time `gorm:"column:run_after;not null;index" json:"run_after"`+ State string `gorm:"column:state;size:16;not null;default:'pending';index" json:"state"`+ ClaimOwner string `gorm:"column:claim_owner;size:64;not null;default:''" json:"claim_owner"`+ ClaimExpiresAt *time.Time `gorm:"column:claim_expires_at;default:null;index" json:"claim_expires_at"`+ ErrorMsg string `gorm:"column:error_msg;type:text;not null" json:"error_msg"`+ CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"created_at"`+ UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime" json:"updated_at"`+}++func (WikiDocumentDirty) TableName() string { return "wiki_document_dirty" }diff --git a/internal/ingestion/component/knowledge_compiler/common/deps.go b/internal/ingestion/component/knowledge_compiler/common/deps.goindex 96111a0b41..5db40c801c 100644--- a/internal/ingestion/component/knowledge_compiler/common/deps.go+++ b/internal/ingestion/component/knowledge_compiler/common/deps.go@@ -131,6 +131,24 @@ type WikiMapVersionStore interface {PutWikiMapVersions(ctx context.Context, versions []WikiMapVersion) error}+// WikiMapActiveState is the mutable pointer to the chunk/hash MAP versions and+// page plan used by the latest successful document Wiki compile. Payload is+// owned by the wiki variant so the storage layer remains schema-independent.+type WikiMapActiveState struct {+ Key string+ TenantID string+ DatasetID string+ DocumentID string+ Payload []byte+}++// WikiMapActiveStateStore persists the active MAP/plan snapshot separately+// from immutable WikiMapVersion history.+type WikiMapActiveStateStore interface {+ GetWikiMapActiveState(ctx context.Context, tenantID, datasetID, key string) ([]byte, error)+ PutWikiMapActiveState(ctx context.Context, state WikiMapActiveState) error+}+// RedisClient is injected only for the datasetnav variant (M8).type RedisClient interface {// Minimal lock surface; full API added in M8.diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.goindex 31445d54e6..6f803cb9ba 100644--- a/internal/ingestion/component/knowledge_compiler/common/types.go+++ b/internal/ingestion/component/knowledge_compiler/common/types.go@@ -138,6 +138,9 @@ type Product struct {type Outputs struct {Products []ProductDuplicatesDropped int+ AffectedPageSlugs []string+ RemovedPageSlugs []string+ WikiActiveStates []WikiMapActiveState}// ParseParam builds a Param from the DSL params map. The variant is NOT takendiff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex 29696603cd..05e3a07978 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -210,6 +210,9 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, ino.Products[i].Variant = variant}out.Products = append(out.Products, o.Products...)+ out.AffectedPageSlugs = append(out.AffectedPageSlugs, o.AffectedPageSlugs...)+ out.RemovedPageSlugs = append(out.RemovedPageSlugs, o.RemovedPageSlugs...)+ out.WikiActiveStates = append(out.WikiActiveStates, o.WikiActiveStates...)}// Convert the compiled products into chunk-aligned docs (matching@@ -246,7 +249,52 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inzap.Int("wiki_relation", relationCount),zap.Int("chunk_docs", len(compiled)),)- return mergeChunks(inputs, compiled), nil+ result := mergeChunks(inputs, compiled)+ if len(out.AffectedPageSlugs) > 0 {+ result["wiki_affected_slugs"] = uniqueSorted(out.AffectedPageSlugs)+ }+ if len(out.RemovedPageSlugs) > 0 {+ result["wiki_removed_slugs"] = uniqueSorted(out.RemovedPageSlugs)+ }+ if len(out.WikiActiveStates) > 0 {+ result["wiki_active_map_states"] = wikiActiveStateValues(out.WikiActiveStates)+ }+ return result, nil+}++// wikiActiveStateValues keeps the component output checkpoint-safe. Pipeline+// node outputs cross an eino serialization boundary, so package-specific Go+// structs must not escape in map[string]any values.+func wikiActiveStateValues(states []common.WikiMapActiveState) []map[string]any {+ values := make([]map[string]any, 0, len(states))+ for _, state := range states {+ values = append(values, map[string]any{+ "key": state.Key,+ "tenant_id": state.TenantID,+ "dataset_id": state.DatasetID,+ "document_id": state.DocumentID,+ "payload": string(state.Payload),+ })+ }+ return values+}++func uniqueSorted(values []string) []string {+ seen := make(map[string]struct{}, len(values))+ result := make([]string, 0, len(values))+ for _, value := range values {+ value = strings.TrimSpace(value)+ if value == "" {+ continue+ }+ if _, ok := seen[value]; ok {+ continue+ }+ seen[value] = struct{}{}+ result = append(result, value)+ }+ sort.Strings(result)+ return result}// resolveTemplateSpecs resolves the configured compilation template spec(s) todiff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex 21420a5805..b5369a9e15 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -19,6 +19,18 @@ import ("gorm.io/gorm")+func TestWikiActiveStateValuesAreCheckpointSafe(t *testing.T) {+ values := wikiActiveStateValues([]common.WikiMapActiveState{{+ Key: "state-1", TenantID: "tenant-1", DatasetID: "kb-1", DocumentID: "doc-1", Payload: []byte(`{"plan":[]}`),+ }})+ if len(values) != 1 || values[0]["payload"] != `{"plan":[]}` {+ t.Fatalf("checkpoint values = %#v", values)+ }+ if _, err := json.Marshal(map[string]any{"wiki_active_map_states": values}); err != nil {+ t.Fatalf("marshal checkpoint-safe active states: %v", err)+ }+}+// mockChat answers the structure variant's three LLM call shapes under the// Python-aligned items contract: node extraction (entity items), edge// extraction (relation items constrained to Known Entities), and mergediff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex de9db1975c..717cbac419 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -17,7 +17,9 @@ import ("regexp""sort""strings"+ "sync"+ "ragflow/internal/agent/runtime"appcommon "ragflow/internal/common""ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/ingestion/component/knowledge_compiler/structure"@@ -61,6 +63,8 @@ func runBatches(ctx context.Context, jobs []func() error) error {// hitting the output-token limit and truncating the payload.const wikiMapTokenBudget = 2048+const wikiRefineProgressStep = 5+// wikiMapMaxTokens derives the extraction output budget from the model's// context length and the per-batch input budget: once the batch has consumed// wikiMapTokenBudget input tokens, the rest of the window is handed to the@@ -105,6 +109,15 @@ type wikiPipeline struct {// hard cap; testable and reported for observability.planCapacityExcluded intneighborCache map[string]*common.WikiPageCandidate+ incremental bool+ activeStateKey string+ previousActiveState wikiMapActiveSnapshot+ nextActiveState wikiMapActiveSnapshot+ affectedPageSlugs map[string]struct{}+ removedPageSlugs []string+ mapChanged bool+ affectedTerms map[string]struct{}+ pendingActiveState *common.WikiMapActiveState}type wikiExtract struct {@@ -116,6 +129,16 @@ type wikiExtract struct {Mode string `json:"mode,omitempty"`}+type wikiMapActiveSnapshot struct {+ Chunks map[string]wikiMapActiveChunk `json:"chunks"`+ Plan []wikiPlanPage `json:"plan"`+}++type wikiMapActiveChunk struct {+ Key string `json:"key"`+ Extract wikiExtract `json:"extract"`+}+type wikiEntity struct {Name string `json:"name"`Type string `json:"type"`@@ -219,16 +242,21 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commozap.Int("chunks", len(inputs.Chunks)),zap.Bool("chat_ready", deps.Chat != nil),zap.Bool("embed_ready", deps.Embed != nil))+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("Wiki Started: input_chunks=%d", len(inputs.Chunks)))p := &wikiPipeline{- ctx: ctx,- deps: deps,- param: param,- inputs: inputs,- docID: docID,- tenantID: deps.TenantID,- datasetID: deps.DatasetID,- llmID: llmID,- neighborCache: make(map[string]*common.WikiPageCandidate),+ ctx: ctx,+ deps: deps,+ param: param,+ inputs: inputs,+ docID: docID,+ tenantID: deps.TenantID,+ datasetID: deps.DatasetID,+ llmID: llmID,+ neighborCache: make(map[string]*common.WikiPageCandidate),+ affectedPageSlugs: make(map[string]struct{}),+ }+ if incremental, ok := inputs.VariantSpecific["wiki_incremental"].(bool); ok {+ p.incremental = incremental}if err := p.run(); err != nil {appcommon.Error("wiki: Run pipeline failed", err,@@ -244,6 +272,8 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commozap.Int("refined_pages", len(p.pages)))products := buildWikiPageProducts(p.tenantID, p.docID, p.pages)+ pageProducts, sectionProducts := countWikiProducts(products)+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("Wiki Done: input_chunks=%d output_products=%d wiki_pages=%d wiki_sections=%d", len(inputs.Chunks), len(products), pageProducts, sectionProducts))appcommon.Info("wiki: built page products",zap.String("dataset_id", deps.DatasetID),zap.String("doc_id", docID),@@ -304,6 +334,11 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoout := common.Outputs{Products: prod,DuplicatesDropped: stats.DuplicatesDropped,+ AffectedPageSlugs: sortedStringSet(p.affectedPageSlugs),+ RemovedPageSlugs: append([]string(nil), p.removedPageSlugs...),+ }+ if p.pendingActiveState != nil {+ out.WikiActiveStates = append(out.WikiActiveStates, *p.pendingActiveState)}return out, nil}@@ -325,8 +360,10 @@ func (p *wikiPipeline) runKey() string {func (p *wikiPipeline) run() error {if len(p.inputs.Chunks) == 0 {+ runtime.ReportProgressMessage(p.ctx, "Compiler", "Wiki Empty: input_chunks=0 output_products=0")return nil}+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki MAP Started: input_chunks=%d", len(p.inputs.Chunks)))if err := p.runMap(); err != nil {return err}@@ -335,6 +372,7 @@ func (p *wikiPipeline) run() error {zap.String("doc_id", p.runKey()),zap.Int("batches", len(p.mapExtracts)),zap.Int("raw_entities", countExtractEntities(p.mapExtracts)))+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki MAP Done: input_chunks=%d output_batches=%d entities=%d concepts=%d claims=%d relations=%d topics=%d", len(p.inputs.Chunks), len(p.mapExtracts), countExtractEntities(p.mapExtracts), countExtractConcepts(p.mapExtracts), countExtractClaims(p.mapExtracts), countExtractRelations(p.mapExtracts), countExtractTopics(p.mapExtracts)))appcommon.Debug("wiki: MAP intermediate",zap.String("dataset_id", p.datasetID),zap.String("doc_id", p.runKey()),@@ -343,6 +381,7 @@ func (p *wikiPipeline) run() error {zap.Strings("concepts", extractConceptDebug(p.mapExtracts)),zap.Strings("claims", extractClaimDebug(p.mapExtracts)),zap.Strings("relations", extractRelationDebug(p.mapExtracts)))+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REDUCE Started: input_entities=%d input_concepts=%d input_claims=%d input_relations=%d", countExtractEntities(p.mapExtracts), countExtractConcepts(p.mapExtracts), countExtractClaims(p.mapExtracts), countExtractRelations(p.mapExtracts)))reduced := reduceExtracts(p.mapExtracts)p.reduced = reducedappcommon.Debug("wiki: REDUCE exact done",@@ -370,12 +409,14 @@ func (p *wikiPipeline) run() error {zap.String("doc_id", p.runKey()),zap.Int("entities", len(p.reduced.Entities)),zap.Int("claims", len(p.reduced.Claims)))+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REDUCE Done: output_entities=%d output_concepts=%d output_claims=%d output_relations=%d", len(p.reduced.Entities), len(p.reduced.Concepts), len(p.reduced.Claims), len(p.reduced.Relations)))// PLAN (B-mode only): LLM-based page plan + reconcile against existing pages.// Mode A (param.PlanEnabled() == false, the default) skips the planner entirely — every// extracted entity/concept becomes its own flat page (1 identity = 1 page), so// the wiki is a flat encyclopedia without PLAN-grouped pages. See buildModeAPlan.var plan wikiPlanvar err error+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki PLAN Started: input_entities=%d input_concepts=%d input_relations=%d", len(p.reduced.Entities), len(p.reduced.Concepts), len(p.reduced.Relations)))if p.param.PlanEnabled() {plan, err = p.runPlan()if err != nil {@@ -389,11 +430,13 @@ func (p *wikiPipeline) run() error {appcommon.Info("wiki: PLAN (A-mode flat) done", zap.String("dataset_id", p.datasetID), zap.String("doc_id", p.runKey()), zap.Int("plan_pages", len(plan.Pages)))}p.plan = plan+ p.selectAffectedPages(plan.Pages)appcommon.Info("wiki: PLAN done",zap.String("dataset_id", p.datasetID),zap.String("doc_id", p.runKey()),zap.Int("plan_pages", len(plan.Pages)),zap.Int("capacity_excluded", p.planCapacityExcluded))+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki PLAN Done: output_pages=%d topics=%d", len(plan.Pages), countPlanTopics(plan.Pages)))appcommon.Debug("wiki: PLAN intermediate",zap.String("dataset_id", p.datasetID),zap.String("doc_id", p.runKey()),@@ -402,11 +445,20 @@ func (p *wikiPipeline) run() error {zap.Int("plan_pages", len(plan.Pages)),zap.Int("capacity_excluded", p.planCapacityExcluded),zap.Strings("pages", planPagesDebug(plan.Pages)))+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REFINE Started: input_pages=%d", len(p.plan.Pages)))pages, err := p.runRefine()if err != nil {return err}p.pages = pages+ activeState, err := p.buildActiveMapState(plan.Pages)+ if err != nil {+ return err+ }+ if activeState != nil {+ p.pendingActiveState = activeState+ }+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REFINE Done: input_pages=%d output_pages=%d", len(p.plan.Pages), len(pages)))appcommon.Info("wiki: REFINE done",zap.String("dataset_id", p.datasetID),zap.String("doc_id", p.runKey()),@@ -428,6 +480,62 @@ func countExtractEntities(extracts []wikiExtract) int {return n}+func countExtractConcepts(extracts []wikiExtract) int {+ n := 0+ for _, e := range extracts {+ n += len(e.Concepts)+ }+ return n+}++func countExtractClaims(extracts []wikiExtract) int {+ n := 0+ for _, e := range extracts {+ n += len(e.Claims)+ }+ return n+}++func countExtractRelations(extracts []wikiExtract) int {+ n := 0+ for _, e := range extracts {+ n += len(e.Relations)+ }+ return n+}++func countExtractTopics(extracts []wikiExtract) int {+ n := 0+ for _, e := range extracts {+ n += len(e.Topics)+ }+ return n+}++func countPlanTopics(pages []wikiPlanPage) int {+ topics := make(map[string]struct{})+ for _, page := range pages {+ topic := strings.TrimSpace(page.Topic)+ if topic != "" {+ topics[topic] = struct{}{}+ }+ }+ return len(topics)+}++func countWikiProducts(products []common.Product) (pages, sections int) {+ for _, product := range products {+ kind, _ := product.Meta["kind"].(string)+ switch kind {+ case "page":+ pages+++ case "section":+ sections+++ }+ }+ return pages, sections+}+// --- Debug helpers: compact per-item descriptors for the stage-intermediate// logs in run(). They deliberately record only identity-level fields (name /// slug / type / aliases / endpoints), never full content or embedding vectors,@@ -714,14 +822,23 @@ func (p *wikiPipeline) runLegacyPlan() (wikiPlan, error) {// order regardless of completion order (no concurrent append to a shared slice).// When no submitter is wired, jobs run serially (historic default).func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {- pages := normalizeWikiPlanPages(p.plan.Pages, p.reduced)- if len(pages) == 0 {+ allPages := normalizeWikiPlanPages(p.plan.Pages, p.reduced)+ if len(allPages) == 0 {return nil, nil}+ pages := allPages+ if p.incremental {+ pages = pages[:0:0]+ for _, page := range allPages {+ if _, affected := p.affectedPageSlugs[page.Slug]; affected {+ pages = append(pages, page)+ }+ }+ }pageTitles := map[string]string{}slugToPageType := map[string]string{}- allPlanSlugs := make([]string, 0, len(pages))- for _, page := range pages {+ allPlanSlugs := make([]string, 0, len(allPages))+ for _, page := range allPages {if page.Slug == "" {continue}@@ -739,6 +856,8 @@ func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {results := make([]wikiPageResult, len(pages))jobs := make([]func() error, 0, len(pages))+ var progressMu sync.Mutex+ completed := 0for i, planItem := range pages {i, planItem := i, planItemjobs = append(jobs, func() error {@@ -750,6 +869,12 @@ func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {return err}results[i] = res+ progressMu.Lock()+ completed+++ if shouldReportRefineProgress(completed, len(pages)) {+ runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REFINE Progress: completed_pages=%d total_pages=%d", completed, len(pages)))+ }+ progressMu.Unlock()return nil})}@@ -759,6 +884,10 @@ func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {return results, nil}+func shouldReportRefineProgress(completed, total int) bool {+ return completed > 0 && total > 0 && (completed%wikiRefineProgressStep == 0 || completed == total)+}+// runRefinePage generates one page result from a normalized plan page. UPDATE// merge, evidence assembly, and source-context building are unchanged; this is// the per-page unit that runRefine fans out.diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state.gonew file mode 100644index 0000000000..aa497772c8--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state.go@@ -0,0 +1,158 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package wiki++import (+ "encoding/json"+ "fmt"+ "sort"+ "strings"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++func (p *wikiPipeline) addAffectedExtractTerms(extract wikiExtract) {+ if p.affectedTerms == nil {+ p.affectedTerms = make(map[string]struct{})+ }+ add := func(value string) {+ value = normalizedWikiTerm(value)+ if value != "" {+ p.affectedTerms[value] = struct{}{}+ }+ }+ for _, entity := range extract.Entities {+ add(entity.Name)+ for _, alias := range entity.Aliases {+ add(alias)+ }+ }+ for _, concept := range extract.Concepts {+ add(concept.Term)+ }+ for _, claim := range extract.Claims {+ add(claim.Subject)+ }+ for _, relation := range extract.Relations {+ add(relation.From)+ add(relation.To)+ }+ for _, topic := range extract.Topics {+ add(topic)+ }+}++func (p *wikiPipeline) selectAffectedPages(current []wikiPlanPage) {+ if p.affectedPageSlugs == nil {+ p.affectedPageSlugs = make(map[string]struct{})+ }+ if !p.incremental || len(p.previousActiveState.Chunks) == 0 {+ for _, page := range current {+ if page.Slug != "" {+ p.affectedPageSlugs[page.Slug] = struct{}{}+ }+ }+ return+ }+ if !p.mapChanged {+ return+ }+ previousBySlug := planPagesBySlug(p.previousActiveState.Plan)+ currentBySlug := planPagesBySlug(current)+ for slug, page := range currentBySlug {+ previous, existed := previousBySlug[slug]+ if !existed || planPageSignature(previous) != planPageSignature(page) || p.pageTouchesAffectedTerms(page) {+ p.affectedPageSlugs[slug] = struct{}{}+ }+ }+ for slug, page := range previousBySlug {+ if _, remains := currentBySlug[slug]; !remains {+ p.affectedPageSlugs[slug] = struct{}{}+ p.removedPageSlugs = append(p.removedPageSlugs, slug)+ continue+ }+ if p.pageTouchesAffectedTerms(page) {+ p.affectedPageSlugs[slug] = struct{}{}+ }+ }+ sort.Strings(p.removedPageSlugs)+}++func (p *wikiPipeline) pageTouchesAffectedTerms(page wikiPlanPage) bool {+ if topic := normalizedWikiTerm(page.Topic); topic != "" {+ if _, ok := p.affectedTerms[topic]; ok {+ return true+ }+ }+ for _, name := range page.EntityNames {+ if _, ok := p.affectedTerms[normalizedWikiTerm(name)]; ok {+ return true+ }+ }+ return false+}++func (p *wikiPipeline) buildActiveMapState(plan []wikiPlanPage) (*common.WikiMapActiveState, error) {+ if p.activeStateKey == "" || p.deps.WikiMapVersions == nil {+ return nil, nil+ }+ p.nextActiveState.Plan = append([]wikiPlanPage(nil), plan...)+ payload, err := json.Marshal(p.nextActiveState)+ if err != nil {+ return nil, fmt.Errorf("wiki: encode active MAP state: %w", err)+ }+ return &common.WikiMapActiveState{+ Key: p.activeStateKey,+ TenantID: p.tenantID,+ DatasetID: p.datasetID,+ DocumentID: p.docID,+ Payload: payload,+ }, nil+}++func planPagesBySlug(pages []wikiPlanPage) map[string]wikiPlanPage {+ result := make(map[string]wikiPlanPage, len(pages))+ for _, page := range pages {+ if page.Slug != "" {+ result[page.Slug] = page+ }+ }+ return result+}++func planPageSignature(page wikiPlanPage) string {+ payload, _ := json.Marshal(struct {+ Title string+ PageType string+ Topic string+ EntityNames []string+ RelatedKB []string+ }{page.Title, page.PageType, page.Topic, page.EntityNames, page.RelatedKB})+ return string(payload)+}++func normalizedWikiTerm(value string) string {+ return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(value)), " "))+}++func sortedStringSet(values map[string]struct{}) []string {+ result := make([]string, 0, len(values))+ for value := range values {+ result = append(result, value)+ }+ sort.Strings(result)+ return result+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state_test.gonew file mode 100644index 0000000000..be68b0f81c--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_incremental_state_test.go@@ -0,0 +1,83 @@+package wiki++import "testing"++func TestSelectAffectedPagesIncludesOldAndNewMembership(t *testing.T) {+ pipeline := &wikiPipeline{+ incremental: true,+ mapChanged: true,+ affectedTerms: map[string]struct{}{+ "alpha": {},+ },+ affectedPageSlugs: map[string]struct{}{},+ previousActiveState: wikiMapActiveSnapshot{+ Chunks: map[string]wikiMapActiveChunk{"chunk-1": {Key: "old"}},+ Plan: []wikiPlanPage{+ {Slug: "topic/old", PageType: "topic", EntityNames: []string{"Alpha"}},+ {Slug: "entity/stable", PageType: "entity", EntityNames: []string{"Stable"}},+ },+ },+ }+ current := []wikiPlanPage{+ {Slug: "topic/new", PageType: "topic", EntityNames: []string{"Alpha"}},+ {Slug: "entity/stable", PageType: "entity", EntityNames: []string{"Stable"}},+ }++ pipeline.selectAffectedPages(current)++ if _, ok := pipeline.affectedPageSlugs["topic/old"]; !ok {+ t.Fatal("old membership page was not affected")+ }+ if _, ok := pipeline.affectedPageSlugs["topic/new"]; !ok {+ t.Fatal("new membership page was not affected")+ }+ if _, ok := pipeline.affectedPageSlugs["entity/stable"]; ok {+ t.Fatal("unrelated stable page was affected")+ }+ if len(pipeline.removedPageSlugs) != 1 || pipeline.removedPageSlugs[0] != "topic/old" {+ t.Fatalf("removed pages = %#v, want topic/old", pipeline.removedPageSlugs)+ }+}++func TestSelectAffectedPagesNoChunkDeltaIsNoOp(t *testing.T) {+ pipeline := &wikiPipeline{+ incremental: true,+ mapChanged: false,+ affectedPageSlugs: map[string]struct{}{},+ previousActiveState: wikiMapActiveSnapshot{+ Chunks: map[string]wikiMapActiveChunk{"chunk-1": {Key: "same"}},+ },+ }+ pipeline.selectAffectedPages([]wikiPlanPage{{Slug: "entity/alpha", EntityNames: []string{"Alpha"}}})+ if len(pipeline.affectedPageSlugs) != 0 {+ t.Fatalf("affected pages = %#v, want none", pipeline.affectedPageSlugs)+ }+}++func TestSelectAffectedPagesMatchesOnlyChangedTopic(t *testing.T) {+ pipeline := &wikiPipeline{+ incremental: true,+ mapChanged: true,+ affectedTerms: map[string]struct{}{+ "history": {},+ },+ affectedPageSlugs: map[string]struct{}{},+ previousActiveState: wikiMapActiveSnapshot{+ Chunks: map[string]wikiMapActiveChunk{"chunk-1": {Key: "old"}},+ Plan: []wikiPlanPage{+ {Slug: "topic/history", PageType: "topic", Topic: "History"},+ {Slug: "topic/science", PageType: "topic", Topic: "Science"},+ },+ },+ }+ current := append([]wikiPlanPage(nil), pipeline.previousActiveState.Plan...)++ pipeline.selectAffectedPages(current)++ if _, ok := pipeline.affectedPageSlugs["topic/history"]; !ok {+ t.Fatal("changed topic page was not selected")+ }+ if _, ok := pipeline.affectedPageSlugs["topic/science"]; ok {+ t.Fatal("unrelated topic page was selected")+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.goindex 47deab1bce..cf4eafbbcc 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go@@ -42,6 +42,24 @@ func (p *wikiPipeline) runVersionedMap() error {return err}llmFingerprint := wikiMapHash(p.llmID)+ p.activeStateKey = wikiMapHash(strings.Join([]string{+ p.tenantID, p.datasetID, p.docID, templateFingerprint, llmFingerprint, "active",+ }, "\x00"))+ p.previousActiveState = wikiMapActiveSnapshot{Chunks: map[string]wikiMapActiveChunk{}}+ if activeStore, ok := p.deps.WikiMapVersions.(common.WikiMapActiveStateStore); ok {+ payload, err := activeStore.GetWikiMapActiveState(p.ctx, p.tenantID, p.datasetID, p.activeStateKey)+ if err != nil {+ return fmt.Errorf("wiki: load active MAP state: %w", err)+ }+ if len(payload) > 0 {+ if err := json.Unmarshal(payload, &p.previousActiveState); err != nil {+ return fmt.Errorf("wiki: decode active MAP state: %w", err)+ }+ }+ }+ if p.previousActiveState.Chunks == nil {+ p.previousActiveState.Chunks = map[string]wikiMapActiveChunk{}+ }versions := make([]wikiMapChunkVersion, 0, len(p.inputs.Chunks))keys := make([]string, 0, len(p.inputs.Chunks))@@ -149,6 +167,32 @@ func (p *wikiPipeline) runVersionedMap() error {}}p.mapExtracts = append(p.mapExtracts, uncachedExtracts...)+ p.nextActiveState = wikiMapActiveSnapshot{Chunks: make(map[string]wikiMapActiveChunk, len(versions))}+ currentChunkIDs := make(map[string]struct{}, len(versions))+ for _, version := range versions {+ currentChunkIDs[version.chunk.ID] = struct{}{}+ extract := extracts[version.index]+ p.nextActiveState.Chunks[version.chunk.ID] = wikiMapActiveChunk{Key: version.key, Extract: extract}+ previous, existed := p.previousActiveState.Chunks[version.chunk.ID]+ if !existed || previous.Key != version.key {+ p.mapChanged = true+ p.addAffectedExtractTerms(previous.Extract)+ p.addAffectedExtractTerms(extract)+ }+ }+ for chunkID, previous := range p.previousActiveState.Chunks {+ if _, active := currentChunkIDs[chunkID]; active {+ continue+ }+ p.mapChanged = true+ p.addAffectedExtractTerms(previous.Extract)+ }+ if len(uncacheable) > 0 {+ p.mapChanged = true+ for _, extract := range uncachedExtracts {+ p.addAffectedExtractTerms(extract)+ }+ }return nil}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_refine_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_refine_test.goindex 67710cf020..af7b76166f 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_refine_test.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_refine_test.go@@ -154,3 +154,12 @@ func TestRunRefine_CancelledCtxAborts(t *testing.T) {t.Fatalf("runRefine err = nil, want context cancelled")}}++func TestShouldReportRefineProgress(t *testing.T) {+ for completed := 1; completed <= 12; completed++ {+ want := completed == 5 || completed == 10 || completed == 12+ if got := shouldReportRefineProgress(completed, 12); got != want {+ t.Fatalf("completed=%d: got %t, want %t", completed, got, want)+ }+ }+}diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex dcdcc5bcf2..1b09d1ec2a 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -344,6 +344,7 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map"output_format": "chunks","chunks": schema.ChunkDocsToMaps(chunks),}+ copyPipelineControlValues(out, inputs)// Embedding requires a KB: the embedder (and its embd_id) is configured// on the knowledgebase, so without kb_id there is nothing to resolve@@ -369,6 +370,14 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mapreturn out, nil}+func copyPipelineControlValues(output, input map[string]any) {+ for _, key := range []string{"wiki_active_map_states"} {+ if value, exists := input[key]; exists {+ output[key] = value+ }+ }+}+func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, embeddingModel, name string, chunks []schema.ChunkDoc) ([]schema.ChunkDoc, int, error) {if len(chunks) == 0 {return chunks, 0, nildiff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex 985dd48981..5637323acd 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -24,6 +24,7 @@ import ("context""encoding/json""os"+ "reflect""strings""sync/atomic""testing"@@ -264,6 +265,15 @@ func TestTokenizerComponent_InputsOutputs_NonEmpty(t *testing.T) {}}+func TestCopyPipelineControlValuesPreservesWikiActiveState(t *testing.T) {+ states := []map[string]any{{"key": "state-1", "payload": `{"plan":[]}`}}+ output := map[string]any{"chunks": []any{}}+ copyPipelineControlValues(output, map[string]any{"wiki_active_map_states": states})+ if !reflect.DeepEqual(output["wiki_active_map_states"], states) {+ t.Fatalf("wiki active states = %#v, want %#v", output["wiki_active_map_states"], states)+ }+}+// TestTokenizerComponent_NewTokenizerComponent_Defaults verifies// the Python default param values propagate.func TestTokenizerComponent_NewTokenizerComponent_Defaults(t *testing.T) {diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex ebac9d9d4a..ba55469cd9 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -47,10 +47,11 @@ var kcDB *gorm.DB// broker — is the scheduling system of record and the source of same-KB// serialization.type Consumer struct {- scheduler Claimer- reader Reader- writer Writer- factory DeduperFactory+ scheduler Claimer+ reader Reader+ writer Writer+ factory DeduperFactory+ contributions wikiContributionStorettl time.Durationheartbeat time.Duration@@ -84,6 +85,7 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {reader: engineReader{eng: engine.Get()},writer: engineWriter{eng: engine.Get()},factory: defaultDeduperFactory,+ contributions: newWikiContributionStore(engine.Get()),ttl: 2 * time.Minute,heartbeat: 20 * time.Second,pollInterval: 2 * time.Second,@@ -168,6 +170,10 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {if paused {return}+ if err := startDatasetCompileLog(ctx, cr.TenantID, datasetID, cr.Token, cr.Entries); err != nil {+ common.Warn("knowledge_compile: failed to create dataset ingestion log",+ zap.String("dataset_id", datasetID), zap.Error(err))+ }// Heartbeat refreshes the claim TTL while we process; a failed touch means// the lease was taken over (or reclaimed) and we must abort without acking.@@ -206,10 +212,18 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {case <-ctx.Done():close(stopHb)<-done+ if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, common.STOPPED, "Wiki dataset compilation stopped during shutdown", -1); err != nil {+ common.Warn("knowledge_compile: failed to finalize stopped dataset ingestion log",+ zap.String("dataset_id", datasetID), zap.Error(err))+ }return // graceful shutdown: leave inflight for reclamation, do not ackcase <-hbFailed:close(stopHb)<-done+ if err := finishDatasetCompileLog(context.WithoutCancel(ctx), cr.Token, common.STOPPED, "Wiki dataset compilation lease was lost and will be retried", -1); err != nil {+ common.Warn("knowledge_compile: failed to finalize stopped dataset ingestion log",+ zap.String("dataset_id", datasetID), zap.Error(err))+ }return // lease lost: do not ack; sweeper/redelivery reprocesses (idempotent)case <-done:close(stopHb)@@ -230,12 +244,44 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {common.Warn("knowledge_compile: failed to record error_msg",zap.String("dataset_id", datasetID), zap.Error(err))}+ if err := finishDatasetCompileLog(ctx, cr.Token, common.FAILED, batchErr.Error(), -1); err != nil {+ common.Warn("knowledge_compile: failed to finalize dataset ingestion log",+ zap.String("dataset_id", datasetID), zap.Error(err))+ }return}if _, err := c.scheduler.Ack(ctx, datasetID, cr.Token, cr.Entries); err != nil {common.Warn("knowledge_compile: ack failed",zap.String("dataset_id", datasetID), zap.Error(err))+ if logErr := finishDatasetCompileLog(ctx, cr.Token, common.FAILED, "Failed to acknowledge completed Wiki compilation: "+err.Error(), -1); logErr != nil {+ common.Warn("knowledge_compile: failed to finalize dataset ingestion log",+ zap.String("dataset_id", datasetID), zap.Error(logErr))+ }+ return+ }+ if err := finishDatasetCompileLog(ctx, cr.Token, common.COMPLETED, "Wiki dataset compilation completed", 1); err != nil {+ common.Warn("knowledge_compile: failed to finalize dataset ingestion log",+ zap.String("dataset_id", datasetID), zap.Error(err))+ }+}++func (c *Consumer) reportProgress(ctx context.Context, tenant, datasetID, token string, progress float64, phase, message string) {+ if err := c.scheduler.UpdateProgress(ctx, datasetID, token, progress, phase, message); err != nil {+ common.Warn("knowledge_compile: failed to update progress",+ zap.String("dataset_id", datasetID),+ zap.String("phase", phase),+ zap.Error(err))}+ if err := updateDatasetCompileLog(ctx, token, progress, message); err != nil {+ common.Warn("knowledge_compile: failed to update dataset ingestion log",+ zap.String("dataset_id", datasetID), zap.String("phase", phase), zap.Error(err))+ }+ common.Info("knowledge_compile: progress",+ zap.String("dataset_id", datasetID),+ zap.String("tenant_id", tenant),+ zap.Float64("progress", progress),+ zap.String("phase", phase),+ zap.String("message", message))}var errClaimSuperseded = errors.New("knowledge_compile: claim superseded by rewrite")@@ -317,7 +363,10 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)if derr := c.writer.DeleteMergedForVariant(ctx, tenant, kb, nil); derr != nil {return derr}- return c.writer.DropWikiGraph(ctx, tenant, kb)+ if derr := c.writer.DropWikiGraph(ctx, tenant, kb); derr != nil {+ return derr+ }+ return c.contributions.ClearDataset(ctx, tenant, kb)}); err != nil {return fmt.Errorf("knowledge_compile: rebuild clear all variants + graph: %w", err)}@@ -444,6 +493,7 @@ func filterWikiPageCandidates(candidates []kccommon.Product) []kccommon.Product// returns an error if any reader/dedup/writer step fails so the caller can// leave the batch for reclamation instead of acking dropped work.func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, entries []BacklogEntry) error {+ c.reportProgress(ctx, tenant, kb, token, 0.05, "classifying", fmt.Sprintf("Classifying %d scheduling entries", len(entries)))common.Info("knowledge_compile: processing claimed batch",zap.String("dataset_id", kb),zap.String("tenant_id", tenant),@@ -527,6 +577,33 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ezap.Strings("entries", entryDetails))if len(deleted) == 0 && len(completed) == 0 {+ c.reportProgress(ctx, tenant, kb, token, 1, "completed", "No document products require merging")+ return nil+ }+ deletedSet := make(map[string]bool, len(deleted))+ for _, d := range deleted {+ deletedSet[d] = true+ }+ wikiDiff, err := c.prepareWikiContributionDiff(ctx, tenant, kb, completed, deleted)+ if err != nil {+ return err+ }+ c.reportProgress(ctx, tenant, kb, token, 0.10, "comparing_wiki_contributions",+ fmt.Sprintf("Comparing Wiki contributions: %d affected page(s)", len(wikiDiff.affectedKeys)))+ wikiOnly := len(completed) > 0+ for _, entry := range completed {+ if len(entry.Variants) != 1 || !variantsContain(entry.Variants, "wiki") {+ wikiOnly = false+ break+ }+ }+ if len(deleted) == 0 && wikiOnly && len(wikiDiff.affectedKeys) == 0 {+ c.mu.Lock()+ for _, docID := range pendingTombClear {+ delete(c.tombs[kb], docID)+ }+ c.mu.Unlock()+ c.reportProgress(ctx, tenant, kb, token, 1, "completed", "Wiki contributions are unchanged; no dataset pages require updating")return nil}@@ -546,11 +623,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ereturn "deduper factory returned nil"}()))}-- deletedSet := make(map[string]bool, len(deleted))- for _, d := range deleted {- deletedSet[d] = true- }+ c.reportProgress(ctx, tenant, kb, token, 0.12, "deleting", fmt.Sprintf("Deleting products for %d document(s)", len(deleted)))// --- Deletion (two sequential DocEngine calls, no in-memory load) ---// Deleted wins regardless of batch order, so we process deletions first.@@ -614,21 +687,39 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// completed and deleted is a stale tombstone: the deletion wins, so we skip// its completion.var incoming []kccommon.Product+ c.reportProgress(ctx, tenant, kb, token, 0.28, "loading_products", fmt.Sprintf("Loading products for %d document(s)", len(completed)))for _, e := range completed {if deletedSet[e.DocID] {continue}- docProducts, err := c.reader.LoadDocProducts(ctx, tenant, kb, e.DocID)- if err != nil {- return err- }+ docProducts := wikiDiff.productsByDoc[e.DocID]// A0-4: the event's Variants are the authoritative dispatch signal — route// ONLY the products whose variant the compiler actually produced for this// doc. A doc re-compiled under a new template may still hold stale// doc-level rows of an earlier variant in storage; without this gate those// would leak into the nav/structure paths despite the event asking only// for, say, wiki. Legacy events with empty Variants keep all products.- incoming = append(incoming, productsForVariants(docProducts, e.Variants)...)+ selected := productsForVariants(docProducts, e.Variants)+ for _, product := range selected {+ if product.Variant != kccommon.VariantWiki {+ incoming = append(incoming, product)+ }+ }+ }+ if len(wikiDiff.affectedKeys) > 0 {+ reader, ok := c.reader.(documentWikiPageReader)+ if !ok {+ return fmt.Errorf("knowledge_compile: reader does not support Wiki contribution loading")+ }+ pages, err := reader.LoadDocumentWikiPagesBySlugs(ctx, tenant, kb, sortedStringSet(wikiDiff.affectedSlugs))+ if err != nil {+ return err+ }+ for _, page := range pages {+ if _, affected := wikiDiff.affectedKeys[wikiPageMergeKey(page)]; affected {+ incoming = append(incoming, page)+ }+ }}// A0-4 dispatch: route tree/structure products to the dataset-navigation tree@@ -638,6 +729,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// so both upsert their summary into the cross-document nav tree; wiki products// continue to the page-specific evidence merge.navIn := navInputFromProducts(kb, incoming)+ c.reportProgress(ctx, tenant, kb, token, 0.40, "merging_navigation", fmt.Sprintf("Merging navigation products: %d", len(navIn)))if len(navIn) > 0 {ns := nav.GetNavService()if ns == nil {@@ -661,6 +753,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e}return err}+ c.reportProgress(ctx, tenant, kb, token, 0.50, "merging_structure", fmt.Sprintf("Merging structure products: %d", len(incoming)))// wiki_incremental port (M1): the dataset-level merge only processes wiki// PAGES. A wiki doc yields both page and section products (Meta.kind@@ -671,13 +764,31 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// contract. Legacy rows whose kind is empty are derived in the Reader; only// truly page-kind wiki products proceed.candidates := filterWikiPageCandidates(incoming)+ c.reportProgress(ctx, tenant, kb, token, 0.54, "filtering_candidates", fmt.Sprintf("Filtering Wiki page candidates: %d", len(candidates)))entityMerged, topicCandidates, err := c.mergeEntityModeCandidates(ctx, tenant, kb, candidates)if err != nil {return err}+ // A Wiki page is identified by its page type and canonical slug. Do not use+ // content KNN or an LLM to decide whether two pages are the same: different+ // slugs are distinct pages, while equal slugs are deterministically folded.+ topicMerged, err := c.mergeWikiProductsBySlug(ctx, tenant, kb, topicCandidates)+ if err != nil {+ return err+ }+ entityMerged = append(entityMerged, topicMerged...)+ if len(wikiDiff.affectedKeys) > 0 {+ if err := c.deleteMissingScopedWikiPages(ctx, tenant, kb, token, wikiDiff.affectedKeys, entityMerged); err != nil {+ return err+ }+ }+ c.reportProgress(ctx, tenant, kb, token, 0.58, "merging_by_slug", fmt.Sprintf("Merging Wiki pages by slug: %d", len(entityMerged)))// Entity pages have stable page identity and must bypass topic-mode KNN and// Page-specific entity/concept merging. Only topic-mode pages continue below.- candidates = topicCandidates+ // The slug-only merge above has already handled every Wiki page candidate.+ // Leave the legacy generic dedup path with no candidates so it cannot apply+ // content similarity or LLM merge decisions to Wiki pages.+ candidates = nil// In-memory dedup among the completed batch first.candidates, dedupErr := deduper.Dedup(ctx, candidates)if dedupErr != nil {@@ -837,6 +948,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, eif err := runCompilerJobs(ctx, jobs); err != nil {return err}+ c.reportProgress(ctx, tenant, kb, token, 0.72, "routing_pages", fmt.Sprintf("Routing %d Wiki pages against existing products", len(candidates)))// KNN only narrows topic-page candidates. When the hit is a topic page,// require an explicit topic match or an LLM topic-route decision before@@ -892,6 +1004,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, evar newMerged []kccommon.ProductnewMerged = append(newMerged, entityMerged...)if len(groupsByID) > 0 {+ c.reportProgress(ctx, tenant, kb, token, 0.82, "llm_merge", fmt.Sprintf("Merging %d candidate groups with the LLM", len(groupsByID)))// Diagnostics: summarize the KNN groups before the LLM merge decision so// the reader can see which existing merged rows were hit and by how many// candidates (e.g. whether a "吕布" candidate hit an existing 吕布 row and@@ -943,6 +1056,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, emergedFinal = append(mergedFinal, unmatched...)mergedFinal, staleTopicIDs := mergeTopicProducts(tenant, kb, mergedFinal)mergedFinal = refreshWikiProductVectors(ctx, tenant, mergedFinal)+ c.reportProgress(ctx, tenant, kb, token, 0.90, "writing_products", fmt.Sprintf("Writing %d merged products", len(mergedFinal)))if err := c.withWriteLock(ctx, kb, token, func() error {if err := c.writer.WriteMerged(ctx, tenant, kb, mergedFinal); err != nil {return err@@ -973,6 +1087,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// then just drops any stale graph rows). The graph is reconstructible, so the// cost of an occasional no-op reprojection is accepted. It is also a locked// destructive side effect for the same TOCTOU reasons as WriteMerged.+ c.reportProgress(ctx, tenant, kb, token, 0.97, "projecting_graph", "Projecting the Wiki graph")if err := c.withWriteLock(ctx, kb, token, func() error {return c.writer.ProjectWikiGraph(ctx, tenant, kb)}); err != nil {@@ -982,6 +1097,9 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e}return err}+ if err := c.commitWikiContributions(ctx, tenant, kb, wikiDiff.currentByDoc, deleted); err != nil {+ return err+ }// All merge and delete paths succeeded: commit the staged tombstone// clears. These are only now persisted so a failed batch leaves c.tombs@@ -996,6 +1114,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ezap.Int("completed_docs", len(completed)),zap.Int("deleted_docs", len(deleted)),zap.Int("merged_rows_written", len(mergedFinal)))+ c.reportProgress(ctx, tenant, kb, token, 1, "completed", fmt.Sprintf("Wiki compilation completed: %d merged products", len(mergedFinal)))return nil}@@ -1007,13 +1126,12 @@ func copyMeta(input map[string]any) map[string]any {return output}-// mergeEntityModeCandidates handles the stage-3 entity page contract. Entity-// and concept pages are keyed by their deterministic slug, not by embedding-// proximity. A batch may contain evidence for the same page from multiple-// documents; fold it before WriteMerged and retain the existing page body when-// it is already present. Topic pages remain on the KNN/LLM route below.+// mergeEntityModeCandidates handles entity/concept pages by their deterministic+// page identity. A batch may contain evidence for the same page from multiple+// documents; fold it before WriteMerged. Topic pages are returned for the same+// slug-only path.func (c *Consumer) mergeEntityModeCandidates(ctx context.Context, tenant, kb string, candidates []kccommon.Product) ([]kccommon.Product, []kccommon.Product, error) {- groups := make(map[string][]kccommon.Product)+ var entityCandidates []kccommon.Productvar topic []kccommon.Productfor _, candidate := range candidates {pageType := strings.ToLower(strings.TrimSpace(metaString(candidate.Meta, "page_type")))@@ -1021,71 +1139,180 @@ func (c *Consumer) mergeEntityModeCandidates(ctx context.Context, tenant, kb strtopic = append(topic, candidate)continue}- key := datasetLevelID(tenant, kb, candidate)- groups[key] = append(groups[key], candidate)+ entityCandidates = append(entityCandidates, candidate)}- if len(groups) == 0 {- return nil, topic, nil+ merged, err := c.mergeWikiProductsBySlug(ctx, tenant, kb, entityCandidates)+ return merged, topic, err+}++// mergeWikiProductsBySlug folds document-level Wiki pages into the dataset+// level using only page_type + slug. Equal identities are merged; different+// slugs remain independent pages. A single document page is already a complete+// Wiki page and passes through unchanged; only groups containing multiple+// document pages need an LLM rewrite.+func (c *Consumer) mergeWikiProductsBySlug(ctx context.Context, tenant, kb string, candidates []kccommon.Product) ([]kccommon.Product, error) {+ if len(candidates) == 0 {+ return nil, nil}- keys := make([]string, 0, len(groups))- for key := range groups {- keys = append(keys, key)++ groups := make(map[string][]kccommon.Product)+ order := make([]string, 0, len(candidates))+ for _, candidate := range candidates {+ key := wikiPageMergeKey(candidate)+ if key == "" {+ // A malformed page without a slug cannot be safely merged with another+ // page. Keep it isolated so it remains visible for diagnosis.+ key = "id\x00" + candidate.ID+ }+ if _, exists := groups[key]; !exists {+ order = append(order, key)+ }+ groups[key] = append(groups[key], candidate)}- sort.Strings(keys)- reader, canLoadExisting := c.reader.(mergedProductReader)- deps, depsErr := kccommon.ResolveDeps(tenant, defaultLLMID, defaultEmbedding)- merged := make([]kccommon.Product, 0, len(keys))- for _, key := range keys {++ merged := make([]kccommon.Product, 0, len(order))+ rewriteIndexes := make([]int, 0, len(order))+ for _, key := range order {var current kccommon.Product- if canLoadExisting {- var err error- current, err = reader.LoadMergedProduct(ctx, tenant, kb, key)- if err != nil {- return nil, nil, err- }- }+ exists := falsefor _, candidate := range groups[key] {candidate.Merged = truecandidate.DocID = kb- if current.ID == "" {+ if !exists || current.ID == "" {current = candidate+ exists = true+ continue+ }+ if strings.EqualFold(strings.TrimSpace(metaString(current.Meta, "page_type")), "topic") {+ current = mergeTopicPage(current, candidate)} else {- if depsErr == nil {- current = c.synthesizeEntityPage(ctx, deps, current, candidate)- } else {- current = wikiEntityMerge(current, candidate)- }+ current = wikiEntityMerge(current, candidate)}}if current.ID != "" {current.Merged = truecurrent.DocID = kbmerged = append(merged, current)+ if len(groups[key]) > 1 {+ rewriteIndexes = append(rewriteIndexes, len(merged)-1)+ }}}- return merged, topic, nil+ if err := rewriteMergedWikiPages(ctx, tenant, merged, rewriteIndexes); err != nil {+ return nil, err+ }+ return merged, nil}-// synthesizeEntityPage updates one entity page from the complete old and new-// evidence. Unlike topic-mode replacement, the model receives both bodies and-// is required to preserve every factual block. If the model cannot be resolved-// (for example in an offline test), the deterministic block union remains safe.-func (c *Consumer) synthesizeEntityPage(ctx context.Context, deps kccommon.Deps, existing, incoming kccommon.Product) kccommon.Product {- merged := wikiEntityMerge(existing, incoming)+const wikiPageRewriteSystemPrompt = `You are the final editor for a knowledge-base Wiki page.+Rewrite the supplied evidence into one coherent Markdown page.+Preserve every factual detail from the evidence, including names, dates, numbers,+relationships, quoted statements, and links. Do not invent facts or remove+information merely to shorten the page. Organize duplicated or fragmented+evidence into a clear structure. Return Markdown only, without commentary about+the editing process.`++// rewriteMergedWikiPages lets the LLM compose the final dataset-level page+// after deterministic slug grouping has selected the evidence that belongs to+// that page. The calls are independent and use the shared compiler pool.+func rewriteMergedWikiPages(ctx context.Context, tenant string, pages []kccommon.Product, indexes []int) error {+ if len(indexes) == 0 {+ return nil+ }+ deps, err := kccommon.ResolveDeps(tenant, defaultLLMID, defaultEmbedding)+ if err != nil {+ return fmt.Errorf("resolve Wiki page rewrite dependencies: %w", err)+ }if deps.Chat == nil {- return merged+ return fmt.Errorf("Wiki page rewrite chat model is unavailable")}- resp, err := deps.Chat.Chat(ctx, kccommon.ChatRequest{- LLMID: defaultLLMID,- SystemPrompt: "You are an entity wiki editor. Rewrite the complete page from both evidence versions. Preserve every supported factual statement, do not invent facts, and return only Markdown.",- UserPrompt: fmt.Sprintf("Entity page: %s\n\nEXISTING PAGE:\n%s\n\nNEW EVIDENCE PAGE:\n%s\n\nWrite the complete merged page. Do not omit facts that appear in either version.",- metaString(existing.Meta, "title"), existing.Content, incoming.Content),- })- if err != nil || resp == nil || strings.TrimSpace(resp.Content) == "" {- return merged+ maxTokens := deps.ModelMaxOutput+ if maxTokens <= 0 {+ maxTokens = 4096+ }+ jobs := make([]CompilerJob, 0, len(indexes))+ for _, idx := range indexes {+ idx := idx+ jobs = append(jobs, func() error {+ page := pages[idx]+ prompt := fmt.Sprintf("PAGE TYPE: %s\nTITLE: %s\nTOPIC: %s\nENTITIES: %s\nSUMMARY: %s\n\nEVIDENCE MARKDOWN:\n%s",+ metaString(page.Meta, "page_type"),+ metaString(page.Meta, "title"),+ metaString(page.Meta, "topic"),+ strings.Join(metaStringSlice(page.Meta, "entity_names"), ", "),+ metaString(page.Meta, "summary"),+ page.Content)+ response, err := deps.Chat.Chat(ctx, kccommon.ChatRequest{+ LLMID: defaultLLMID,+ SystemPrompt: wikiPageRewriteSystemPrompt,+ UserPrompt: prompt,+ Temperature: floatPtr(0.1),+ MaxTokens: &maxTokens,+ })+ if err != nil {+ return fmt.Errorf("rewrite Wiki page %q: %w", metaString(page.Meta, "slug"), err)+ }+ if response == nil {+ return fmt.Errorf("rewrite Wiki page %q returned no response", metaString(page.Meta, "slug"))+ }+ content := strings.TrimSpace(response.Content)+ if content == "" {+ return fmt.Errorf("rewrite Wiki page %q returned empty content", metaString(page.Meta, "slug"))+ }+ pages[idx].Content = content+ return nil+ })+ }+ return SubmitCompilerJobs(ctx, jobs)+}++func floatPtr(value float64) *float64 { return &value }++func wikiPageMergeKey(product kccommon.Product) string {+ slug := strings.TrimSpace(metaString(product.Meta, "slug"))+ if slug == "" {+ return ""}- merged.Content = strings.TrimSpace(resp.Content)- return merged+ pageType := strings.ToLower(strings.TrimSpace(metaString(product.Meta, "page_type")))+ return pageType + "\x00" + strings.ToLower(strings.Join(strings.Fields(slug), " "))+}++type mergedWikiPageDeleter interface {+ DeleteMergedWikiPages(ctx context.Context, tenant, kb string, ids []string) error+}++func (c *Consumer) deleteMissingScopedWikiPages(ctx context.Context, tenant, kb, token string, affected map[string]struct{}, current []kccommon.Product) error {+ reader, readerOK := c.reader.(mergedWikiPageReader)+ deleter, writerOK := c.writer.(mergedWikiPageDeleter)+ if !readerOK || !writerOK {+ return fmt.Errorf("knowledge_compile: scoped Wiki cleanup is unsupported")+ }+ present := make(map[string]struct{}, len(current))+ for _, page := range current {+ if key := wikiPageMergeKey(page); key != "" {+ present[key] = struct{}{}+ }+ }+ existing, err := reader.LoadMergedWikiPages(ctx, tenant, kb)+ if err != nil {+ return err+ }+ ids := make([]string, 0)+ for _, page := range existing {+ key := wikiPageMergeKey(page)+ if _, scoped := affected[key]; !scoped {+ continue+ }+ if _, remains := present[key]; !remains && page.ID != "" {+ ids = append(ids, page.ID)+ }+ }+ if len(ids) == 0 {+ return nil+ }+ return c.withWriteLock(ctx, kb, token, func() error {+ return deleter.DeleteMergedWikiPages(ctx, tenant, kb, ids)+ })}func refreshWikiProductVectors(ctx context.Context, tenant string, products []kccommon.Product) []kccommon.Product {@@ -1114,6 +1341,24 @@ func productContentsForEmbedding(products []kccommon.Product) []string {return contents}+func variantsContain(variants []string, wanted string) bool {+ for _, variant := range variants {+ if variant == wanted {+ return true+ }+ }+ return false+}++func sortedStringSet(values map[string]struct{}) []string {+ result := make([]string, 0, len(values))+ for value := range values {+ result = append(result, value)+ }+ sort.Strings(result)+ return result+}+// candidateIdentity returns a compact identity string for a product, preferring// the wiki slug (full "<page_type>/<slug>") when present, else the id/doc_id,// so KNN/dedup diagnostics can tie a candidate to the page it belongs to.diff --git a/internal/ingestion/knowledge_compile/dataset_log.go b/internal/ingestion/knowledge_compile/dataset_log.gonew file mode 100644index 0000000000..c271273281--- /dev/null+++ b/internal/ingestion/knowledge_compile/dataset_log.go@@ -0,0 +1,122 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package knowledge_compile++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/entity"+)++const datasetLogDocumentID = "graph_raptor_x"++func datasetCompileLogID(claimToken string) string {+ sum := sha256.Sum256([]byte("wiki-dataset-log\x00" + claimToken))+ return hex.EncodeToString(sum[:16])+}++func startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimToken string, entries []BacklogEntry) error {+ if kcDB == nil || claimToken == "" {+ return nil+ }+ now := time.Now()+ status := "1"+ message := fmt.Sprintf("Created automatic Wiki dataset task for %d document event(s)", len(entries))+ entryData := make([]any, 0, len(entries))+ for _, entry := range entries {+ entryData = append(entryData, map[string]any{+ "doc_id": entry.DocID,+ "event_type": entry.EventType,+ "variants": entry.Variants,+ })+ }+ log := entity.PipelineOperationLog{+ ID: datasetCompileLogID(claimToken),+ DocumentID: datasetLogDocumentID,+ TenantID: tenantID,+ KbID: datasetID,+ ParserID: "wiki",+ DocumentName: "Wiki",+ DocumentSuffix: "",+ DocumentType: "dataset",+ SourceFrom: "knowledgebase",+ Progress: 0,+ ProgressMsg: &message,+ ProcessBeginAt: &now,+ DSL: entity.JSONMap{"entries": entryData},+ TaskType: string(entity.PipelineTaskTypeWiki),+ OperationStatus: common.RUNNING,+ Status: &status,+ }+ return kcDB.WithContext(ctx).Where("id = ?", log.ID).FirstOrCreate(&log).Error+}++func updateDatasetCompileLog(ctx context.Context, claimToken string, progress float64, message string) error {+ if kcDB == nil || claimToken == "" {+ return nil+ }+ logID := datasetCompileLogID(claimToken)+ var log entity.PipelineOperationLog+ if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {+ return err+ }+ progressMessage := ""+ if log.ProgressMsg != nil {+ progressMessage = *log.ProgressMsg+ }+ progressMessage = appendProgressMessage(progressMessage, message)+ duration := log.ProcessDuration+ if log.ProcessBeginAt != nil {+ duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())+ }+ return kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{+ "progress": progress,+ "progress_msg": progressMessage,+ "process_duration": duration,+ "operation_status": common.RUNNING,+ }).Error+}++func finishDatasetCompileLog(ctx context.Context, claimToken, operationStatus, message string, progress float64) error {+ if kcDB == nil || claimToken == "" {+ return nil+ }+ logID := datasetCompileLogID(claimToken)+ var log entity.PipelineOperationLog+ if err := kcDB.WithContext(ctx).Where("id = ?", logID).First(&log).Error; err != nil {+ return err+ }+ progressMessage := ""+ if log.ProgressMsg != nil {+ progressMessage = *log.ProgressMsg+ }+ progressMessage = appendProgressMessage(progressMessage, message)+ duration := log.ProcessDuration+ if log.ProcessBeginAt != nil {+ duration = max(0, time.Since(*log.ProcessBeginAt).Seconds())+ }+ return kcDB.WithContext(ctx).Model(&entity.PipelineOperationLog{}).Where("id = ?", logID).Updates(map[string]any{+ "progress": progress,+ "progress_msg": progressMessage,+ "process_duration": duration,+ "operation_status": operationStatus,+ }).Error+}diff --git a/internal/ingestion/knowledge_compile/dataset_log_test.go b/internal/ingestion/knowledge_compile/dataset_log_test.gonew file mode 100644index 0000000000..09ccefaf61--- /dev/null+++ b/internal/ingestion/knowledge_compile/dataset_log_test.go@@ -0,0 +1,50 @@+package knowledge_compile++import (+ "strings"+ "testing"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"++ "ragflow/internal/common"+ "ragflow/internal/entity"+)++func TestDatasetCompileLogLifecycle(t *testing.T) {+ db, err := gorm.Open(sqlite.Open("file:dataset-log?mode=memory&cache=shared"), &gorm.Config{})+ if err != nil {+ t.Fatalf("open sqlite: %v", err)+ }+ if err := db.AutoMigrate(&entity.PipelineOperationLog{}); err != nil {+ t.Fatalf("migrate pipeline log: %v", err)+ }+ oldDB := kcDB+ kcDB = db+ t.Cleanup(func() { kcDB = oldDB })++ entries := []BacklogEntry{{DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"}}}+ if err := startDatasetCompileLog(t.Context(), "tenant-1", "kb-1", "claim-1", entries); err != nil {+ t.Fatalf("start dataset log: %v", err)+ }+ if err := updateDatasetCompileLog(t.Context(), "claim-1", 0.5, "Comparing Wiki contributions: 2 affected page(s)"); err != nil {+ t.Fatalf("update dataset log: %v", err)+ }+ if err := finishDatasetCompileLog(t.Context(), "claim-1", common.COMPLETED, "Wiki dataset compilation completed", 1); err != nil {+ t.Fatalf("finish dataset log: %v", err)+ }++ var log entity.PipelineOperationLog+ if err := db.Where("id = ?", datasetCompileLogID("claim-1")).First(&log).Error; err != nil {+ t.Fatalf("load dataset log: %v", err)+ }+ if log.DocumentID != datasetLogDocumentID || log.TaskType != string(entity.PipelineTaskTypeWiki) {+ t.Fatalf("unexpected dataset log identity: %+v", log)+ }+ if log.OperationStatus != common.COMPLETED || log.Progress != 1 {+ t.Fatalf("unexpected final state: status=%s progress=%v", log.OperationStatus, log.Progress)+ }+ if log.ProgressMsg == nil || !strings.Contains(*log.ProgressMsg, "2 affected page(s)") || !strings.Contains(*log.ProgressMsg, "completed") {+ t.Fatalf("progress messages were not persisted: %v", log.ProgressMsg)+ }+}diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex fb3b6364b3..2cc3c1127f 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -65,6 +65,10 @@ type mergedWikiPageReader interface {LoadMergedWikiPages(ctx context.Context, tenant, kb string) ([]kccommon.Product, error)}+type documentWikiPageReader interface {+ LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant, kb string, slugs []string) ([]kccommon.Product, error)+}+// engineReader loads the per-document compiled products through the global// DocEngine (§11.6 step 1, §11.7 incremental re-dedup). It depends on the// process-wide DocEngine obtained via engine.Get(); the engine abstraction owns@@ -256,6 +260,46 @@ func (r engineReader) LoadMergedWikiPages(ctx context.Context, tenant, kb stringreturn pages, nil}+func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant, kb string, slugs []string) ([]kccommon.Product, error) {+ eng := r.eng+ if eng == nil {+ eng = engine.Get()+ }+ if eng == nil || len(slugs) == 0 {+ return nil, nil+ }+ filter := map[string]interface{}{+ "available_int": 0,+ "compile_kwd": compileKwdWikiPage,+ "slug_kwd": slugs,+ }+ var pages []kccommon.Product+ for offset := 0; ; offset += loadDocProductsLimit {+ res, err := eng.Search(ctx, &types.SearchRequest{+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)}, KbIDs: []string{kb},+ Limit: loadDocProductsLimit, Offset: offset,+ OrderBy: (&types.OrderByExpr{}).Asc("id"),+ SelectFields: append(append([]string(nil), compiledSelectFields...), wikiSelectFields...),+ Filter: filter,+ })+ if err != nil {+ return nil, err+ }+ if res == nil || len(res.Chunks) == 0 {+ break+ }+ for _, row := range res.Chunks {+ if product, ok := productFromChunkMap(row, tenant, kccommon.VariantWiki); ok && metaString(product.Meta, "kind") == "page" {+ pages = append(pages, product)+ }+ }+ if len(res.Chunks) < loadDocProductsLimit {+ break+ }+ }+ return pages, nil+}+// productFromChunkMap reconstructs a kccommon.Product from a stored compiled// chunk document. It reads the payload from kc_payload (falling back to// content_with_weight) and the embedding from the q_<dim>_vec column.diff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.goindex c2c4f14ad1..1701410994 100644--- a/internal/ingestion/knowledge_compile/scheduler.go+++ b/internal/ingestion/knowledge_compile/scheduler.go@@ -123,6 +123,9 @@ type Claimer interface {// live, so a stale worker whose lease was reclaimed cannot overwrite the// status of the worker that took over.SetError(ctx context.Context, datasetID, token, errMsg string) error+ // UpdateProgress records the current batch phase and progress while the+ // claim is live. Updates from a stale worker are ignored by token.+ UpdateProgress(ctx context.Context, datasetID, token string, progress float64, phase, message string) error// CancelInflight drops the live claim for a dataset back into the backlog so// a rewrite can start from a clean index. The token is intentionally NOT@@ -214,7 +217,7 @@ func (s *mysqlScheduler) Provision(ctx context.Context) error {// each other's backlog), and the notify is always paired with the append so a// producer never needs a separate Notify call.func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {- return s.publish(ctx, tenantID, datasetID, docID, eventType, variants)+ return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})}// loadRow fetches the dataset scheduling row (no lock). Returns nil (no error)@@ -231,10 +234,13 @@ func (s *mysqlScheduler) loadRow(ctx context.Context, datasetID string) (*entity}func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {+ return s.publishEntry(ctx, tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})+}++func (s *mysqlScheduler) publishEntry(ctx context.Context, tenantID, datasetID string, entry BacklogEntry) error {if s.db == nil {return nil}- entry := BacklogEntry{DocID: docID, EventType: eventType, Variants: variants}// KnowledgeCompileDataset.dataset_id is the PRIMARY KEY, so the SELECT ... FOR// UPDATE below also takes an InnoDB gap lock for a not-yet-existing dataset;// concurrent publishers for the same dataset serialize on that lock and the@@ -277,8 +283,8 @@ func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docIDcommon.Info("knowledge_compile: published backlog entry",zap.String("dataset_id", datasetID),zap.String("tenant_id", tenantID),- zap.String("doc_id", docID),- zap.String("event_type", eventType))+ zap.String("doc_id", entry.DocID),+ zap.String("event_type", entry.EventType))return s.notify(ctx, datasetID)}@@ -350,6 +356,9 @@ func (s *mysqlScheduler) claimRow(ctx context.Context, tx *gorm.DB, datasetID strow.ClaimExpiresAt = &exprow.State = DatasetStateRunningrow.ErrorMsg = ""+ row.Progress = 0+ row.CurrentPhase = "claiming"+ row.ProgressMsg = ""if err := tx.Save(&row).Error; err != nil {return ClaimResult{}, false, err}@@ -461,6 +470,55 @@ func (s *mysqlScheduler) SetError(ctx context.Context, datasetID, token, errMsgreturn nil}+const progressMessageMaxLength = 3000++func appendProgressMessage(previous, message string) string {+ if message == "" {+ return previous+ }+ if previous == "" {+ if len([]rune(message)) > progressMessageMaxLength {+ return string([]rune(message)[len([]rune(message))-progressMessageMaxLength:])+ }+ return message+ }+ joined := previous + "\n" + message+ if len([]rune(joined)) > progressMessageMaxLength {+ runes := []rune(joined)+ return string(runes[len(runes)-progressMessageMaxLength:])+ }+ return joined+}++// UpdateProgress records a best-effort phase update for the active claim. The+// token check prevents a worker whose lease was reclaimed from overwriting the+// progress of the new owner.+func (s *mysqlScheduler) UpdateProgress(ctx context.Context, datasetID, token string, progress float64, phase, message string) error {+ if s.db == nil {+ return nil+ }+ if progress < 0 {+ progress = 0+ }+ if progress > 1 {+ progress = 1+ }+ return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ var row entity.KnowledgeCompileDataset+ if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).+ Where("dataset_id = ?", datasetID).First(&row).Error; err != nil {+ return err+ }+ if row.ClaimToken != token {+ return nil+ }+ row.Progress = progress+ row.CurrentPhase = phase+ row.ProgressMsg = appendProgressMessage(row.ProgressMsg, message)+ return tx.Save(&row).Error+ })+}+// CancelInflight drops the live claim for a dataset back into the backlog so a// rewrite can start from a clean index. It is a no-op when there is no live// lease. The dropped batch is re-marked pending and is reprocessed after the@@ -679,14 +737,17 @@ func removeEntries(inflight, batch []BacklogEntry) []BacklogEntry {// ---- in-memory scheduler for tests ----type fakeRow struct {- tenant string- backlog []BacklogEntry- inflight []BacklogEntry- owner string- token string- expires *time.Time- state string- errorMsg string+ tenant string+ backlog []BacklogEntry+ inflight []BacklogEntry+ owner string+ token string+ expires *time.Time+ state string+ errorMsg string+ progress float64+ currentPhase string+ progressMsg string// writeMu serializes per-dataset writer side effects and rebuilds, mirroring// the MySQL scheduler's WithDatasetLock row lock.writeMu sync.Mutex@@ -716,10 +777,10 @@ func (f *FakeScheduler) Provision(_ context.Context) error { return nil }// Publish appends one doc event and pushes a notify (same contract as MySQL).func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {- return f.publish(tenantID, datasetID, docID, eventType, variants)+ return f.publishEntry(tenantID, datasetID, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})}-func (f *FakeScheduler) publish(tenantID, datasetID, docID, eventType string, variants []string) error {+func (f *FakeScheduler) publishEntry(tenantID, datasetID string, entry BacklogEntry) error {f.mu.Lock()defer f.mu.Unlock()r, ok := f.rows[datasetID]@@ -730,7 +791,7 @@ func (f *FakeScheduler) publish(tenantID, datasetID, docID, eventType string, vaif r.tenant == "" {r.tenant = tenantID}- r.backlog = append(r.backlog, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})+ r.backlog = append(r.backlog, entry)// Mirror the MySQL scheduler: surface pending unless a worker is running.if r.state != DatasetStateRunning {r.state = DatasetStatePending@@ -767,6 +828,9 @@ func (f *FakeScheduler) Claim(_ context.Context, datasetID string) (ClaimResult,r.expires = &expr.state = DatasetStateRunningr.errorMsg = ""+ r.progress = 0+ r.currentPhase = "claiming"+ r.progressMsg = ""return ClaimResult{DatasetID: datasetID, TenantID: r.tenant, Entries: batch, Token: r.token}, true, nil}@@ -816,6 +880,25 @@ func (f *FakeScheduler) SetError(_ context.Context, datasetID, token, errMsg strreturn nil}+func (f *FakeScheduler) UpdateProgress(_ context.Context, datasetID, token string, progress float64, phase, message string) error {+ f.mu.Lock()+ defer f.mu.Unlock()+ r, ok := f.rows[datasetID]+ if !ok || r.token != token {+ return nil+ }+ if progress < 0 {+ progress = 0+ }+ if progress > 1 {+ progress = 1+ }+ r.progress = progress+ r.currentPhase = phase+ r.progressMsg = appendProgressMessage(r.progressMsg, message)+ return nil+}+// WithDatasetLock executes fn under the per-row write mutex, mirroring the MySQL// scheduler's row-lock semantics: a rebuild and a writer's destructive side// effect on the same dataset are mutually exclusive. claimToken is read underdiff --git a/internal/ingestion/knowledge_compile/scheduler_test.go b/internal/ingestion/knowledge_compile/scheduler_test.goindex 6f749cbcae..fe592b1773 100644--- a/internal/ingestion/knowledge_compile/scheduler_test.go+++ b/internal/ingestion/knowledge_compile/scheduler_test.go@@ -56,6 +56,29 @@ func TestFakeSchedulerReclaimsInterruptedClaim(t *testing.T) {}}+func TestFakeSchedulerProgressIsClaimScoped(t *testing.T) {+ f := NewFakeScheduler()+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), nil); err != nil {+ t.Fatalf("publish: %v", err)+ }+ claim, ok, err := f.Claim(t.Context(), "kb1")+ if err != nil || !ok {+ t.Fatalf("claim: ok=%v err=%v", ok, err)+ }+ if err := f.UpdateProgress(t.Context(), "kb1", claim.Token, 0.5, "routing_pages", "Routing pages"); err != nil {+ t.Fatalf("update progress: %v", err)+ }+ if err := f.UpdateProgress(t.Context(), "kb1", "stale-token", 0.9, "wrong", "Should be ignored"); err != nil {+ t.Fatalf("stale update: %v", err)+ }+ f.mu.Lock()+ row := f.rows["kb1"]+ f.mu.Unlock()+ if row.progress != 0.5 || row.currentPhase != "routing_pages" || row.progressMsg != "Routing pages" {+ t.Fatalf("unexpected progress state: %+v", row)+ }+}+func TestWithWriteLockRejectsSupersededClaim(t *testing.T) {f := NewFakeScheduler()if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}); err != nil {diff --git a/internal/ingestion/knowledge_compile/service.go b/internal/ingestion/knowledge_compile/service.goindex 6377a38269..00541a9acd 100644--- a/internal/ingestion/knowledge_compile/service.go+++ b/internal/ingestion/knowledge_compile/service.go@@ -22,9 +22,11 @@ import ("fmt""time"- "gorm.io/gorm""ragflow/internal/engine"+ "ragflow/internal/entity"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"++ "gorm.io/gorm")// Option configures a Consumer.@@ -77,6 +79,12 @@ func WithWriter(w Writer) Option { return func(c *Consumer) { c.writer = w } }// WithDeduperFactory overrides the per-tenant Deduper factory.func WithDeduperFactory(f DeduperFactory) Option { return func(c *Consumer) { c.factory = f } }+// withWikiContributionStore overrides the durable document-contribution store+// in package tests.+func withWikiContributionStore(store wikiContributionStore) Option {+ return func(c *Consumer) { c.contributions = store }+}+// defaultLLMID / defaultEmbedding are the model ids used when resolving LLM// deps for the dataset-level deduper. Set via SetModelConfig (typically from the// server bootstrap). Empty strings make the factory fall back to the noop@@ -128,11 +136,16 @@ func Provision(ctx context.Context, mq engine.MessageQueue, db *gorm.DB) error {// Bound the startup AutoMigrate so a slow/unreachable DB cannot block// startup indefinitely. The caller's ctx is also honoured (cancelled on// shutdown).+ runCtx := ctxctx, cancel := context.WithTimeout(ctx, 30*time.Second)defer cancel()if err := s.Provision(ctx); err != nil {return err}+ if err := db.WithContext(ctx).AutoMigrate(&entity.WikiDocumentDirty{}); err != nil {+ return err+ }SetScheduler(s)+ go runWikiDirtyWorker(runCtx, db, s.holder)return nil}diff --git a/internal/ingestion/knowledge_compile/topic_merge.go b/internal/ingestion/knowledge_compile/topic_merge.goindex e9e0899ef9..c84042d338 100644--- a/internal/ingestion/knowledge_compile/topic_merge.go+++ b/internal/ingestion/knowledge_compile/topic_merge.go@@ -68,10 +68,9 @@ func topicKey(topic string) string {return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(topic)), " "))}-func topicSlug(topic string) string {- return "topic/" + hashStr(topicKey(topic))-}-+// prepareTopicProduct normalizes topic metadata for legacy routing code. The+// active merge path does not call it to decide identity; slug remains the+// caller-provided stable page identity and is never replaced by a hash.func prepareTopicProduct(product kccommon.Product, topic string) kccommon.Product {product.Meta = copyMeta(product.Meta)topic = strings.TrimSpace(topic)@@ -80,25 +79,19 @@ func prepareTopicProduct(product kccommon.Product, topic string) kccommon.Produc}product.Meta["page_type"] = "topic"product.Meta["topic"] = topic- product.Meta["slug"] = topicSlug(topic)- if productTitle(product) == "" {- product.Meta["title"] = topic- }return product}-// mergeTopicProducts folds pages with the same canonical topic and preserves-// the first dataset-level page identity.+// mergeTopicProducts folds only pages with the same canonical slug and+// preserves the first dataset-level page identity. Similar topics with+// different slugs remain separate pages.func mergeTopicProducts(tenant, kb string, products []kccommon.Product) ([]kccommon.Product, []string) {groups := make(map[string][]kccommon.Product)order := make([]string, 0, len(products))for _, product := range products {- key := candidateIdentity(product)- if isTopicPage(product) {- key = topicKey(productTopic(product))- if key == "" {- key = candidateIdentity(product)- }+ key := wikiPageMergeKey(product)+ if key == "" {+ key = candidateIdentity(product)}if _, exists := groups[key]; !exists {order = append(order, key)@@ -113,19 +106,11 @@ func mergeTopicProducts(tenant, kb string, products []kccommon.Product) ([]kccomcontinue}current := items[0]- if isTopicPage(current) {- existingSlug := metaString(current.Meta, "slug")- current = prepareTopicProduct(current, productTopic(current))- if existingSlug != "" && current.Merged && current.DocID == kb {- current.Meta["slug"] = existingSlug- }- }for _, item := range items[1:] {if item.ID != "" && item.ID != current.ID && item.Merged && item.DocID == kb {stale = append(stale, item.ID)}if isTopicPage(current) {- item = prepareTopicProduct(item, productTopic(current))current = mergeTopicPage(current, item)} else {current = wikiEntityMerge(current, item)@@ -166,7 +151,7 @@ func mergeTopicPage(existing, incoming kccommon.Product) kccommon.Product {merged.Meta["page_type"] = "topic"topic := firstTopicString(metaString(incoming.Meta, "topic"), metaString(existing.Meta, "topic"))merged.Meta["topic"] = topic- merged.Meta["title"] = firstTopicString(metaString(incoming.Meta, "title"), metaString(existing.Meta, "title"), topic)+ merged.Meta["title"] = firstTopicString(metaString(existing.Meta, "title"), metaString(incoming.Meta, "title"), topic)merged.Meta["entity_names"] = unionStrs(metaStringSlice(existing.Meta, "entity_names"), metaStringSlice(incoming.Meta, "entity_names"))return merged}diff --git a/internal/ingestion/knowledge_compile/topic_merge_test.go b/internal/ingestion/knowledge_compile/topic_merge_test.goindex 92a26d610a..0a6dd86eac 100644--- a/internal/ingestion/knowledge_compile/topic_merge_test.go+++ b/internal/ingestion/knowledge_compile/topic_merge_test.go@@ -20,8 +20,8 @@ func TestMergeTopicProductsPreservesEvidenceAndStableTopicIdentity(t *testing.T){ID: "doc-page-2", DocID: "doc-2", Variant: kccommon.VariantWiki, Content: "# History\n\nsecond",Meta: map[string]any{- "kind": "page", "page_type": "topic", "topic": " history ", "title": "History",- "slug": "topic/history-b", "entity_names": []string{"Bob"},+ "kind": "page", "page_type": "topic", "topic": " history ", "title": "History (updated)",+ "slug": "topic/history-a", "entity_names": []string{"Bob"},"source_doc_ids": []string{"doc-2"}, "source_chunk_ids": []string{"chunk-2"},},},@@ -36,8 +36,11 @@ func TestMergeTopicProductsPreservesEvidenceAndStableTopicIdentity(t *testing.T)if !strings.Contains(merged[0].Content, "first") || !strings.Contains(merged[0].Content, "second") {t.Fatalf("merged content lost evidence: %q", merged[0].Content)}- if got := metaString(merged[0].Meta, "slug"); got != "topic/"+hashStr("history") {- t.Fatalf("topic slug = %q, want deterministic topic slug", got)+ if got := metaString(merged[0].Meta, "slug"); got != "topic/history-a" {+ t.Fatalf("topic slug = %q, want existing slug", got)+ }+ if got := metaString(merged[0].Meta, "title"); got != "History" {+ t.Fatalf("topic title = %q, want existing title", got)}if got := metaStringSlice(merged[0].Meta, "entity_names"); len(got) != 2 {t.Fatalf("entity names = %v, want both entities", got)@@ -65,7 +68,7 @@ func TestMergeTopicProductsRemovesSupersededDatasetPage(t *testing.T) {}},}merged, stale := mergeTopicProducts("tenant", "kb", products)- if len(merged) != 1 || len(stale) != 1 || stale[0] != "new-page" {- t.Fatalf("merged=%#v stale=%#v, want one survivor and new-page stale", merged, stale)+ if len(merged) != 2 || len(stale) != 0 {+ t.Fatalf("merged=%#v stale=%#v, want two separate pages", merged, stale)}}diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.gonew file mode 100644index 0000000000..000a4a349a--- /dev/null+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go@@ -0,0 +1,386 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package knowledge_compile++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "sort"+ "strings"+ "sync"++ "ragflow/internal/engine"+ enginetypes "ragflow/internal/engine/types"+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++const wikiContributionCompileKWD = "wiki_contribution_state"++type wikiContributionPage struct {+ Key string `json:"key"`+ Slug string `json:"slug"`+ PageType string `json:"page_type"`+ Signature string `json:"signature"`+}++type wikiDocumentContribution struct {+ DocumentID string `json:"document_id"`+ Pages []wikiContributionPage `json:"pages"`+}++type wikiContributionStore interface {+ Get(ctx context.Context, tenantID, datasetID, documentID string) (wikiDocumentContribution, bool, error)+ Put(ctx context.Context, tenantID, datasetID string, contribution wikiDocumentContribution) error+ Delete(ctx context.Context, tenantID, datasetID, documentID string) error+ ClearDataset(ctx context.Context, tenantID, datasetID string) error+}++type engineWikiContributionStore struct {+ engine engine.DocEngine+}++func newWikiContributionStore(docEngine engine.DocEngine) wikiContributionStore {+ if docEngine == nil {+ return &memoryWikiContributionStore{items: map[string]wikiDocumentContribution{}}+ }+ return &engineWikiContributionStore{engine: docEngine}+}++func (s *engineWikiContributionStore) Get(ctx context.Context, tenantID, datasetID, documentID string) (wikiDocumentContribution, bool, error) {+ key := wikiContributionStateID(datasetID, documentID)+ result, err := s.engine.Search(ctx, &enginetypes.SearchRequest{+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},+ KbIDs: []string{datasetID},+ Limit: 1,+ SelectFields: []string{"id", "compile_kwd", "content_with_weight"},+ Filter: map[string]any{+ "id": []string{key},+ "compile_kwd": wikiContributionCompileKWD,+ "available_int": 0,+ },+ })+ if err != nil {+ return wikiDocumentContribution{}, false, err+ }+ if result == nil || len(result.Chunks) == 0 {+ return wikiDocumentContribution{}, false, nil+ }+ var contribution wikiDocumentContribution+ if err := json.Unmarshal([]byte(mapStoreString(result.Chunks[0]["content_with_weight"])), &contribution); err != nil {+ return wikiDocumentContribution{}, false, fmt.Errorf("decode Wiki contribution for document %s: %w", documentID, err)+ }+ return contribution, true, nil+}++func (s *engineWikiContributionStore) Put(ctx context.Context, tenantID, datasetID string, contribution wikiDocumentContribution) error {+ if contribution.DocumentID == "" {+ return fmt.Errorf("save Wiki contribution: document_id is required")+ }+ payload, err := json.Marshal(contribution)+ if err != nil {+ return fmt.Errorf("encode Wiki contribution: %w", err)+ }+ row := map[string]any{+ "id": wikiContributionStateID(datasetID, contribution.DocumentID),+ "doc_id": "wiki_contribution:" + contribution.DocumentID,+ "tenant_id": tenantID,+ "kb_id": datasetID,+ "compile_kwd": wikiContributionCompileKWD,+ "scope_kwd": "doc",+ "source_doc_ids": []string{contribution.DocumentID},+ "content_with_weight": string(payload),+ "available_int": 0,+ }+ _, err = s.engine.InsertChunks(ctx, []map[string]any{row}, fmt.Sprintf("ragflow_%s", tenantID), datasetID)+ return err+}++func (s *engineWikiContributionStore) Delete(ctx context.Context, tenantID, datasetID, documentID string) error {+ _, err := s.engine.DeleteChunks(ctx, map[string]any{+ "id": []string{wikiContributionStateID(datasetID, documentID)},+ "kb_id": datasetID,+ }, fmt.Sprintf("ragflow_%s", tenantID), datasetID)+ return err+}++func (s *engineWikiContributionStore) ClearDataset(ctx context.Context, tenantID, datasetID string) error {+ _, err := s.engine.DeleteChunks(ctx, map[string]any{+ "kb_id": datasetID,+ "compile_kwd": wikiContributionCompileKWD,+ "available_int": 0,+ }, fmt.Sprintf("ragflow_%s", tenantID), datasetID)+ return err+}++func wikiContributionStateID(datasetID, documentID string) string {+ sum := sha256.Sum256([]byte(datasetID + "\x00" + documentID + "\x00wiki-contribution"))+ return hex.EncodeToString(sum[:])+}++type memoryWikiContributionStore struct {+ mu sync.Mutex+ items map[string]wikiDocumentContribution+}++func (s *memoryWikiContributionStore) Get(_ context.Context, _, datasetID, documentID string) (wikiDocumentContribution, bool, error) {+ s.mu.Lock()+ defer s.mu.Unlock()+ item, ok := s.items[datasetID+"\x00"+documentID]+ return item, ok, nil+}++func (s *memoryWikiContributionStore) Put(_ context.Context, _, datasetID string, contribution wikiDocumentContribution) error {+ s.mu.Lock()+ defer s.mu.Unlock()+ s.items[datasetID+"\x00"+contribution.DocumentID] = contribution+ return nil+}++func (s *memoryWikiContributionStore) Delete(_ context.Context, _, datasetID, documentID string) error {+ s.mu.Lock()+ defer s.mu.Unlock()+ delete(s.items, datasetID+"\x00"+documentID)+ return nil+}++func (s *memoryWikiContributionStore) ClearDataset(_ context.Context, _, datasetID string) error {+ s.mu.Lock()+ defer s.mu.Unlock()+ prefix := datasetID + "\x00"+ for key := range s.items {+ if strings.HasPrefix(key, prefix) {+ delete(s.items, key)+ }+ }+ return nil+}++func buildWikiDocumentContribution(documentID string, products []kccommon.Product) wikiDocumentContribution {+ pagesByKey := make(map[string]wikiContributionPage)+ for _, product := range products {+ if product.Variant != kccommon.VariantWiki || metaString(product.Meta, "kind") != "page" {+ continue+ }+ key := wikiPageMergeKey(product)+ if key == "" {+ continue+ }+ pagesByKey[key] = wikiContributionPage{+ Key: key,+ Slug: strings.TrimSpace(metaString(product.Meta, "slug")),+ PageType: strings.ToLower(strings.TrimSpace(metaString(product.Meta, "page_type"))),+ Signature: wikiContributionSignature(product),+ }+ }+ keys := make([]string, 0, len(pagesByKey))+ for key := range pagesByKey {+ keys = append(keys, key)+ }+ sort.Strings(keys)+ pages := make([]wikiContributionPage, 0, len(keys))+ for _, key := range keys {+ pages = append(pages, pagesByKey[key])+ }+ return wikiDocumentContribution{DocumentID: documentID, Pages: pages}+}++func diffWikiDocumentContribution(previous, current wikiDocumentContribution) (map[string]struct{}, map[string]struct{}) {+ previousByKey := make(map[string]wikiContributionPage, len(previous.Pages))+ for _, page := range previous.Pages {+ previousByKey[page.Key] = page+ }+ currentByKey := make(map[string]wikiContributionPage, len(current.Pages))+ for _, page := range current.Pages {+ currentByKey[page.Key] = page+ }+ affectedKeys := make(map[string]struct{})+ affectedSlugs := make(map[string]struct{})+ for key, page := range currentByKey {+ old, exists := previousByKey[key]+ if exists && old.Signature == page.Signature {+ continue+ }+ affectedKeys[key] = struct{}{}+ if page.Slug != "" {+ affectedSlugs[page.Slug] = struct{}{}+ }+ }+ for key, page := range previousByKey {+ if _, exists := currentByKey[key]; exists {+ continue+ }+ affectedKeys[key] = struct{}{}+ if page.Slug != "" {+ affectedSlugs[page.Slug] = struct{}{}+ }+ }+ return affectedKeys, affectedSlugs+}++func wikiContributionSignature(product kccommon.Product) string {+ normalized := struct {+ Content string+ Title string+ Topic string+ Summary string+ EntityNames []string+ RelatedKBPages []string+ Outlinks []string+ SourceChunkIDs []string+ }{+ Content: strings.TrimSpace(product.Content),+ Title: strings.TrimSpace(metaString(product.Meta, "title")),+ Topic: strings.TrimSpace(metaString(product.Meta, "topic")),+ Summary: strings.TrimSpace(metaString(product.Meta, "summary")),+ EntityNames: sortedUniqueCopy(metaStringSlice(product.Meta, "entity_names")),+ RelatedKBPages: sortedUniqueCopy(metaStringSlice(product.Meta, "related_kb_pages")),+ Outlinks: sortedUniqueCopy(metaStringSlice(product.Meta, "outlinks")),+ SourceChunkIDs: sortedUniqueCopy(metaStringSlice(product.Meta, "source_chunk_ids")),+ }+ payload, _ := json.Marshal(normalized)+ sum := sha256.Sum256(payload)+ return hex.EncodeToString(sum[:])+}++func sortedUniqueCopy(values []string) []string {+ copyValues := append([]string(nil), values...)+ sort.Strings(copyValues)+ out := copyValues[:0]+ for _, value := range copyValues {+ if value == "" || len(out) > 0 && out[len(out)-1] == value {+ continue+ }+ out = append(out, value)+ }+ return out+}++type wikiContributionDiffResult struct {+ productsByDoc map[string][]kccommon.Product+ currentByDoc map[string]wikiDocumentContribution+ affectedKeys map[string]struct{}+ affectedSlugs map[string]struct{}+}++func (c *Consumer) prepareWikiContributionDiff(ctx context.Context, tenantID, datasetID string, completed []BacklogEntry, deleted []string) (wikiContributionDiffResult, error) {+ if c.contributions == nil {+ c.contributions = newWikiContributionStore(nil)+ }+ result := wikiContributionDiffResult{+ productsByDoc: make(map[string][]kccommon.Product, len(completed)),+ currentByDoc: make(map[string]wikiDocumentContribution, len(completed)),+ affectedKeys: map[string]struct{}{},+ affectedSlugs: map[string]struct{}{},+ }+ completedByDoc := make(map[string]BacklogEntry, len(completed))+ for _, entry := range completed {+ completedByDoc[entry.DocID] = entry+ products, err := c.reader.LoadDocProducts(ctx, tenantID, datasetID, entry.DocID)+ if err != nil {+ return wikiContributionDiffResult{}, err+ }+ result.productsByDoc[entry.DocID] = products+ wikiProducts := products+ if len(entry.Variants) > 0 && !variantsContain(entry.Variants, "wiki") {+ wikiProducts = nil+ }+ result.currentByDoc[entry.DocID] = buildWikiDocumentContribution(entry.DocID, wikiProducts)+ }++ allDocIDs := make([]string, 0, len(completed)+len(deleted))+ for docID := range completedByDoc {+ allDocIDs = append(allDocIDs, docID)+ }+ allDocIDs = append(allDocIDs, deleted...)+ sort.Strings(allDocIDs)++ previousByDoc := make(map[string]wikiDocumentContribution, len(allDocIDs))+ missingPrevious := make(map[string]struct{})+ for _, docID := range allDocIDs {+ previous, exists, err := c.contributions.Get(ctx, tenantID, datasetID, docID)+ if err != nil {+ return wikiContributionDiffResult{}, err+ }+ if exists {+ previousByDoc[docID] = previous+ } else {+ missingPrevious[docID] = struct{}{}+ }+ }++ // Upgrade legacy datasets lazily. Before contribution snapshots existed, the+ // only durable ownership record was source_doc_ids on merged pages. Recover+ // enough page identity from those rows to retract a deleted/disabled document+ // correctly on its first post-upgrade event.+ if len(missingPrevious) > 0 {+ if reader, ok := c.reader.(mergedWikiPageReader); ok {+ mergedPages, err := reader.LoadMergedWikiPages(ctx, tenantID, datasetID)+ if err != nil {+ return wikiContributionDiffResult{}, err+ }+ legacyProducts := make(map[string][]kccommon.Product, len(missingPrevious))+ for _, page := range mergedPages {+ for _, docID := range metaStringSlice(page.Meta, "source_doc_ids") {+ if _, wanted := missingPrevious[docID]; wanted {+ legacyProducts[docID] = append(legacyProducts[docID], page)+ }+ }+ }+ for docID, products := range legacyProducts {+ previousByDoc[docID] = buildWikiDocumentContribution(docID, products)+ }+ }+ }++ for _, docID := range allDocIDs {+ previous := previousByDoc[docID]+ current, completedNow := result.currentByDoc[docID]+ if !completedNow {+ current = wikiDocumentContribution{DocumentID: docID}+ }+ keys, slugs := diffWikiDocumentContribution(previous, current)+ for key := range keys {+ result.affectedKeys[key] = struct{}{}+ }+ for slug := range slugs {+ result.affectedSlugs[slug] = struct{}{}+ }+ }+ return result, nil+}++func (c *Consumer) commitWikiContributions(ctx context.Context, tenantID, datasetID string, current map[string]wikiDocumentContribution, deleted []string) error {+ for documentID, contribution := range current {+ if len(contribution.Pages) == 0 {+ if err := c.contributions.Delete(ctx, tenantID, datasetID, documentID); err != nil {+ return fmt.Errorf("delete Wiki contribution for document %s: %w", documentID, err)+ }+ continue+ }+ if err := c.contributions.Put(ctx, tenantID, datasetID, contribution); err != nil {+ return fmt.Errorf("save Wiki contribution for document %s: %w", documentID, err)+ }+ }+ for _, documentID := range deleted {+ if err := c.contributions.Delete(ctx, tenantID, datasetID, documentID); err != nil {+ return fmt.Errorf("delete Wiki contribution for document %s: %w", documentID, err)+ }+ }+ return nil+}diff --git a/internal/ingestion/knowledge_compile/wiki_contribution_test.go b/internal/ingestion/knowledge_compile/wiki_contribution_test.gonew file mode 100644index 0000000000..db4531db14--- /dev/null+++ b/internal/ingestion/knowledge_compile/wiki_contribution_test.go@@ -0,0 +1,92 @@+package knowledge_compile++import (+ "context"+ "testing"++ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++func wikiContributionTestProduct(slug, content string, chunkIDs ...string) kccommon.Product {+ return kccommon.Product{+ ID: "doc-page-" + slug,+ DocID: "doc-1",+ Variant: kccommon.VariantWiki,+ Content: content,+ Meta: map[string]any{+ "kind": "page",+ "page_type": "entity",+ "slug": slug,+ "title": slug,+ "source_doc_ids": []string{"doc-1"},+ "source_chunk_ids": chunkIDs,+ },+ }+}++func TestDiffWikiDocumentContributionIgnoresUnchangedPages(t *testing.T) {+ product := wikiContributionTestProduct("cao-cao", "same content", "chunk-1")+ previous := buildWikiDocumentContribution("doc-1", []kccommon.Product{product})+ current := buildWikiDocumentContribution("doc-1", []kccommon.Product{product})+ keys, slugs := diffWikiDocumentContribution(previous, current)+ if len(keys) != 0 || len(slugs) != 0 {+ t.Fatalf("unchanged contribution produced diff: keys=%v slugs=%v", keys, slugs)+ }+}++func TestDiffWikiDocumentContributionIncludesChangedAndRemovedPages(t *testing.T) {+ previous := buildWikiDocumentContribution("doc-1", []kccommon.Product{+ wikiContributionTestProduct("cao-cao", "old content", "chunk-1"),+ wikiContributionTestProduct("liu-bei", "removed content", "chunk-2"),+ })+ current := buildWikiDocumentContribution("doc-1", []kccommon.Product{+ wikiContributionTestProduct("cao-cao", "new content", "chunk-3"),+ })+ keys, slugs := diffWikiDocumentContribution(previous, current)+ for _, key := range []string{"entity\x00cao-cao", "entity\x00liu-bei"} {+ if _, ok := keys[key]; !ok {+ t.Fatalf("affected key %q missing from %v", key, keys)+ }+ }+ for _, slug := range []string{"cao-cao", "liu-bei"} {+ if _, ok := slugs[slug]; !ok {+ t.Fatalf("affected slug %q missing from %v", slug, slugs)+ }+ }+}++func TestDiffWikiDocumentContributionTreatsDisabledDocumentAsRetraction(t *testing.T) {+ previous := buildWikiDocumentContribution("doc-1", []kccommon.Product{+ wikiContributionTestProduct("cao-cao", "content", "chunk-1"),+ })+ current := wikiDocumentContribution{DocumentID: "doc-1"}+ keys, _ := diffWikiDocumentContribution(previous, current)+ if _, ok := keys["entity\x00cao-cao"]; !ok {+ t.Fatalf("disabled document did not retract its page: %v", keys)+ }+}++func TestProcessBatchSkipsUnchangedWikiOnlyEvent(t *testing.T) {+ product := wikiContributionTestProduct("cao-cao", "same content", "chunk-1")+ store := &memoryWikiContributionStore{items: map[string]wikiDocumentContribution{}}+ if err := store.Put(t.Context(), "tenant-1", "kb-1", buildWikiDocumentContribution("doc-1", []kccommon.Product{product})); err != nil {+ t.Fatalf("seed contribution: %v", err)+ }+ factoryCalled := false+ consumer := NewConsumer(NewFakeScheduler(),+ WithReader(&fakeReader{products: []kccommon.Product{product}}),+ withWikiContributionStore(store),+ WithDeduperFactory(func(string) (Deduper, error) {+ factoryCalled = true+ return nil, nil+ }),+ )+ if err := consumer.processBatch(context.Background(), "tenant-1", "kb-1", "token-1", []BacklogEntry{{+ DocID: "doc-1", EventType: string(EventTypeCompleted), Variants: []string{"wiki"},+ }}); err != nil {+ t.Fatalf("processBatch failed: %v", err)+ }+ if factoryCalled {+ t.Fatal("unchanged Wiki-only event initialized the dataset merge path")+ }+}diff --git a/internal/ingestion/knowledge_compile/wiki_dirty.go b/internal/ingestion/knowledge_compile/wiki_dirty.gonew file mode 100644index 0000000000..eb71fb34a9--- /dev/null+++ b/internal/ingestion/knowledge_compile/wiki_dirty.go@@ -0,0 +1,222 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package knowledge_compile++import (+ "context"+ "encoding/json"+ "fmt"+ "sort"+ "sync"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"++ "go.uber.org/zap"+ "gorm.io/gorm"+ "gorm.io/gorm/clause"+)++const (+ wikiDirtyDebounce = 20 * time.Second+ wikiDirtyLease = 10 * time.Minute+ wikiDirtyPoll = time.Second+)++// WikiDirtyRequest describes one debounced document-level Wiki refresh.+type WikiDirtyRequest struct {+ TenantID string+ DatasetID string+ DocumentID string+ Revision uint64+ AffectedChunkIDs []string+}++// WikiDirtyCompiler regenerates the affected document-level Wiki products.+type WikiDirtyCompiler func(context.Context, WikiDirtyRequest) error++var wikiDirtyCompilerState struct {+ sync.RWMutex+ compiler WikiDirtyCompiler+}++// SetWikiDirtyCompiler installs the document-level compiler owned by the task+// composition root.+func SetWikiDirtyCompiler(compiler WikiDirtyCompiler) {+ wikiDirtyCompilerState.Lock()+ wikiDirtyCompilerState.compiler = compiler+ wikiDirtyCompilerState.Unlock()+}++// MarkWikiDocumentDirty durably applies a trailing-edge debounce. Repeated+// mutations merge chunk ids and postpone execution until twenty seconds after+// the most recent successful mutation.+func MarkWikiDocumentDirty(ctx context.Context, tenantID, datasetID, documentID string, chunkIDs []string) error {+ db := kcDB+ if db == nil {+ db = dao.DB+ }+ if db == nil {+ return fmt.Errorf("mark Wiki document dirty: database is unavailable")+ }+ if tenantID == "" || datasetID == "" || documentID == "" {+ return fmt.Errorf("mark Wiki document dirty: tenant, dataset, and document ids are required")+ }+ runAfter := time.Now().Add(wikiDirtyDebounce)+ return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ row := entity.WikiDocumentDirty{DocumentID: documentID}+ err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).Where("document_id = ?", documentID).First(&row).Error+ if err != nil && err != gorm.ErrRecordNotFound {+ return err+ }+ if err == gorm.ErrRecordNotFound {+ row = entity.WikiDocumentDirty{+ DocumentID: documentID,+ DatasetID: datasetID,+ TenantID: tenantID,+ Revision: 1,+ AffectedChunkIDs: marshalDirtyChunkIDs(chunkIDs),+ RunAfter: runAfter,+ State: entity.WikiDirtyStatePending,+ }+ return tx.Create(&row).Error+ }+ row.TenantID = tenantID+ row.DatasetID = datasetID+ row.Revision+++ row.AffectedChunkIDs = marshalDirtyChunkIDs(append(parseDirtyChunkIDs(row.AffectedChunkIDs), chunkIDs...))+ row.RunAfter = runAfter+ row.State = entity.WikiDirtyStatePending+ row.ErrorMsg = ""+ row.ClaimOwner = ""+ row.ClaimExpiresAt = nil+ return tx.Save(&row).Error+ })+}++func runWikiDirtyWorker(ctx context.Context, db *gorm.DB, owner string) {+ ticker := time.NewTicker(wikiDirtyPoll)+ defer ticker.Stop()+ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ for {+ row, ok, err := claimWikiDirty(ctx, db, owner)+ if err != nil {+ common.Warn("wiki dirty worker: claim failed", zap.Error(err))+ break+ }+ if !ok {+ break+ }+ processWikiDirty(ctx, db, owner, row)+ }+ }+ }+}++func claimWikiDirty(ctx context.Context, db *gorm.DB, owner string) (entity.WikiDocumentDirty, bool, error) {+ var claimed entity.WikiDocumentDirty+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ now := time.Now()+ query := tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}).+ Where("run_after <= ? AND (state = ? OR (state = ? AND claim_expires_at <= ?))",+ now, entity.WikiDirtyStatePending, entity.WikiDirtyStateRunning, now).+ Order("run_after ASC").Limit(1)+ if err := query.First(&claimed).Error; err != nil {+ return err+ }+ expires := now.Add(wikiDirtyLease)+ return tx.Model(&entity.WikiDocumentDirty{}).Where("document_id = ? AND revision = ?", claimed.DocumentID, claimed.Revision).+ Updates(map[string]interface{}{+ "state": entity.WikiDirtyStateRunning,+ "claim_owner": owner,+ "claim_expires_at": expires,+ }).Error+ })+ if err == gorm.ErrRecordNotFound {+ return entity.WikiDocumentDirty{}, false, nil+ }+ return claimed, err == nil, err+}++func processWikiDirty(ctx context.Context, db *gorm.DB, owner string, row entity.WikiDocumentDirty) {+ wikiDirtyCompilerState.RLock()+ compiler := wikiDirtyCompilerState.compiler+ wikiDirtyCompilerState.RUnlock()+ request := WikiDirtyRequest{+ TenantID: row.TenantID,+ DatasetID: row.DatasetID,+ DocumentID: row.DocumentID,+ Revision: row.Revision,+ AffectedChunkIDs: parseDirtyChunkIDs(row.AffectedChunkIDs),+ }+ var err error+ if compiler == nil {+ err = fmt.Errorf("Wiki dirty compiler is not configured")+ } else {+ err = compiler(ctx, request)+ }+ if err == nil {+ db.WithContext(ctx).Where("document_id = ? AND revision = ? AND claim_owner = ?", row.DocumentID, row.Revision, owner).+ Delete(&entity.WikiDocumentDirty{})+ return+ }+ common.Warn("wiki dirty worker: compile failed",+ zap.String("document_id", row.DocumentID), zap.Uint64("revision", row.Revision), zap.Error(err))+ next := time.Now().Add(wikiDirtyDebounce)+ db.WithContext(ctx).Model(&entity.WikiDocumentDirty{}).+ Where("document_id = ? AND revision = ? AND claim_owner = ?", row.DocumentID, row.Revision, owner).+ Updates(map[string]interface{}{+ "state": entity.WikiDirtyStatePending,+ "run_after": next,+ "claim_owner": "",+ "claim_expires_at": nil,+ "error_msg": err.Error(),+ })+}++func parseDirtyChunkIDs(raw string) []string {+ var ids []string+ _ = json.Unmarshal([]byte(raw), &ids)+ return uniqueSortedStrings(ids)+}++func marshalDirtyChunkIDs(ids []string) string {+ payload, _ := json.Marshal(uniqueSortedStrings(ids))+ return string(payload)+}++func uniqueSortedStrings(values []string) []string {+ seen := make(map[string]struct{}, len(values))+ out := make([]string, 0, len(values))+ for _, value := range values {+ if value == "" {+ continue+ }+ if _, ok := seen[value]; ok {+ continue+ }+ seen[value] = struct{}{}+ out = append(out, value)+ }+ sort.Strings(out)+ return out+}diff --git a/internal/ingestion/knowledge_compile/wiki_dirty_test.go b/internal/ingestion/knowledge_compile/wiki_dirty_test.gonew file mode 100644index 0000000000..04cfe59820--- /dev/null+++ b/internal/ingestion/knowledge_compile/wiki_dirty_test.go@@ -0,0 +1,99 @@+package knowledge_compile++import (+ "context"+ "reflect"+ "testing"+ "time"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"++ "ragflow/internal/entity"+)++func TestMarkWikiDocumentDirtyDebouncesAndMergesChunks(t *testing.T) {+ db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})+ if err != nil {+ t.Fatalf("open sqlite: %v", err)+ }+ if err := db.AutoMigrate(&entity.WikiDocumentDirty{}); err != nil {+ t.Fatalf("migrate dirty table: %v", err)+ }+ previousDB := kcDB+ kcDB = db+ t.Cleanup(func() { kcDB = previousDB })++ if err := MarkWikiDocumentDirty(t.Context(), "tenant", "dataset", "document", []string{"chunk-b", "chunk-a"}); err != nil {+ t.Fatalf("first dirty mark: %v", err)+ }+ var first entity.WikiDocumentDirty+ if err := db.First(&first, "document_id = ?", "document").Error; err != nil {+ t.Fatalf("load first dirty mark: %v", err)+ }+ time.Sleep(time.Millisecond)+ if err := MarkWikiDocumentDirty(t.Context(), "tenant", "dataset", "document", []string{"chunk-c", "chunk-b"}); err != nil {+ t.Fatalf("second dirty mark: %v", err)+ }+ var second entity.WikiDocumentDirty+ if err := db.First(&second, "document_id = ?", "document").Error; err != nil {+ t.Fatalf("load second dirty mark: %v", err)+ }++ if second.Revision != 2 {+ t.Fatalf("revision = %d, want 2", second.Revision)+ }+ if !second.RunAfter.After(first.RunAfter) {+ t.Fatalf("run_after was not postponed: first=%s second=%s", first.RunAfter, second.RunAfter)+ }+ if got, want := parseDirtyChunkIDs(second.AffectedChunkIDs), []string{"chunk-a", "chunk-b", "chunk-c"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("affected chunks = %v, want %v", got, want)+ }+ if delay := time.Until(second.RunAfter); delay < wikiDirtyDebounce-time.Second || delay > wikiDirtyDebounce+time.Second {+ t.Fatalf("debounce delay = %s, want about %s", delay, wikiDirtyDebounce)+ }+}++func TestWikiDirtyRevisionPreventsDroppingConcurrentMutation(t *testing.T) {+ db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})+ if err != nil {+ t.Fatalf("open sqlite: %v", err)+ }+ if err := db.AutoMigrate(&entity.WikiDocumentDirty{}); err != nil {+ t.Fatalf("migrate dirty table: %v", err)+ }+ previousDB := kcDB+ kcDB = db+ t.Cleanup(func() { kcDB = previousDB })++ if err := MarkWikiDocumentDirty(t.Context(), "tenant", "dataset", "document", []string{"chunk-a"}); err != nil {+ t.Fatalf("dirty mark: %v", err)+ }+ if err := db.Model(&entity.WikiDocumentDirty{}).Where("document_id = ?", "document").Updates(map[string]any{+ "state": entity.WikiDirtyStateRunning, "claim_owner": "worker",+ }).Error; err != nil {+ t.Fatalf("claim dirty row: %v", err)+ }+ var claimed entity.WikiDocumentDirty+ if err := db.First(&claimed, "document_id = ?", "document").Error; err != nil {+ t.Fatalf("load claimed row: %v", err)+ }++ wikiDirtyCompilerState.Lock()+ previousCompiler := wikiDirtyCompilerState.compiler+ wikiDirtyCompilerState.compiler = func(_ context.Context, _ WikiDirtyRequest) error {+ return MarkWikiDocumentDirty(t.Context(), "tenant", "dataset", "document", []string{"chunk-b"})+ }+ wikiDirtyCompilerState.Unlock()+ t.Cleanup(func() { SetWikiDirtyCompiler(previousCompiler) })++ processWikiDirty(t.Context(), db, "worker", claimed)++ var pending entity.WikiDocumentDirty+ if err := db.First(&pending, "document_id = ?", "document").Error; err != nil {+ t.Fatalf("concurrent mutation was dropped: %v", err)+ }+ if pending.Revision != 2 || pending.State != entity.WikiDirtyStatePending {+ t.Fatalf("pending row = %+v, want revision 2 pending", pending)+ }+}diff --git a/internal/ingestion/knowledge_compile/wiki_map_store.go b/internal/ingestion/knowledge_compile/wiki_map_store.goindex 88e40985e0..85b32358f1 100644--- a/internal/ingestion/knowledge_compile/wiki_map_store.go+++ b/internal/ingestion/knowledge_compile/wiki_map_store.go@@ -29,6 +29,7 @@ import (const (wikiMapExtractCompileKWD = "wiki_map_extract"+ wikiMapActiveCompileKWD = "wiki_map_active"wikiMapStoreBatchSize = 500)@@ -36,6 +37,53 @@ type wikiMapVersionStore struct {engine engine.DocEngine}+func (s *wikiMapVersionStore) GetWikiMapActiveState(ctx context.Context, tenantID, datasetID, key string) ([]byte, error) {+ if s == nil || s.engine == nil {+ return nil, fmt.Errorf("wiki MAP active-state DocStore is not initialized")+ }+ result, err := s.engine.Search(ctx, &types.SearchRequest{+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},+ KbIDs: []string{datasetID},+ Limit: 1,+ SelectFields: []string{"id", "compile_kwd", "content_with_weight"},+ Filter: map[string]interface{}{+ "id": []string{key},+ "compile_kwd": wikiMapActiveCompileKWD,+ "available_int": 0,+ },+ })+ if err != nil || result == nil || len(result.Chunks) == 0 {+ return nil, err+ }+ payload := mapStoreString(result.Chunks[0]["content_with_weight"])+ if payload == "" {+ return nil, nil+ }+ return []byte(payload), nil+}++func (s *wikiMapVersionStore) PutWikiMapActiveState(ctx context.Context, state kccommon.WikiMapActiveState) error {+ if s == nil || s.engine == nil {+ return fmt.Errorf("wiki MAP active-state DocStore is not initialized")+ }+ if state.Key == "" || state.TenantID == "" || state.DatasetID == "" || state.DocumentID == "" {+ return fmt.Errorf("save Wiki MAP active state: key and scope are required")+ }+ row := map[string]interface{}{+ "id": state.Key,+ "doc_id": "wiki_map_active:" + state.DocumentID,+ "tenant_id": state.TenantID,+ "kb_id": state.DatasetID,+ "compile_kwd": wikiMapActiveCompileKWD,+ "scope_kwd": "doc",+ "source_doc_ids": []string{state.DocumentID},+ "content_with_weight": string(state.Payload),+ "available_int": 0,+ }+ _, err := s.engine.InsertChunks(ctx, []map[string]interface{}{row}, fmt.Sprintf("ragflow_%s", state.TenantID), state.DatasetID)+ return err+}+// NewWikiMapVersionStore returns the DocStore-backed immutable Wiki MAP cache.// Its rows remain non-searchable through available_int=0 and the compile// discriminator, while preserving every chunk/hash version for reuse.diff --git a/internal/ingestion/knowledge_compile/wiki_merge.go b/internal/ingestion/knowledge_compile/wiki_merge.goindex 14a89f17c9..4a843d21e7 100644--- a/internal/ingestion/knowledge_compile/wiki_merge.go+++ b/internal/ingestion/knowledge_compile/wiki_merge.go@@ -105,7 +105,11 @@ func splitMarkdownBlocks(markdown string) []string {}// unionWikiProvenance returns a new Meta map based on a (the existing row). The-// candidate b overwrites kind / slug / page_type / title / summary /+// candidate b contributes content metadata, while the existing page identity+// (slug/title/page_type) remains authoritative because pages are merged only+// when their canonical slug is equal. This prevents a retry or another+// document's equivalent page from renaming the dataset-level page.+// The candidate b overwrites kind / summary /// entity_names / related_kb_pages / outlinks. Identity and creation time// (created_at_unix, created_at) stay from a. Source provenance arrays// (source_doc_ids, source_chunk_ids) are unioned and deduped.@@ -116,7 +120,7 @@ func unionWikiProvenance(a, b map[string]any) map[string]any {}// The incoming page replaces current page metadata. Identity and creation// time remain from the existing row.- for _, key := range []string{"slug", "page_type", "title", "summary", "kind"} {+ for _, key := range []string{"summary", "kind"} {if v, ok := b[key]; ok {out[key] = v}diff --git a/internal/ingestion/knowledge_compile/wiki_merge_test.go b/internal/ingestion/knowledge_compile/wiki_merge_test.goindex 4c9cccf8f8..9361e69638 100644--- a/internal/ingestion/knowledge_compile/wiki_merge_test.go+++ b/internal/ingestion/knowledge_compile/wiki_merge_test.go@@ -1,118 +1,37 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-package knowledge_compileimport (- "context"- "strings""testing"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common")-// TestWikiMergePreservesCandidateVector verifies that page evidence merging-// retains both Markdown blocks and uses the candidate embedding for the merged-// body.-func TestWikiMergePreservesCandidateVector(t *testing.T) {- existing := kccommon.Product{- ID: "kb/wiki/alpha",- DocID: "kb",- TenantID: "t1",- Variant: kccommon.VariantWiki,- Content: "# old markdown",- Vector: []float32{1, 0, 0, 0},+func TestMergeWikiProductsBySlugKeepsSingleDocumentPageContent(t *testing.T) {+ page := kccommon.Product{+ ID: "doc-page-1",+ DocID: "doc-1",+ Variant: kccommon.VariantWiki,+ Content: "# Original\n\nDocument-level Wiki content.",Meta: map[string]any{- "kind": "page",- "slug": "page/alpha",- "created_at": "2024-01-01",+ "kind": "page",+ "page_type": "entity",+ "slug": "entity/alpha",+ "title": "Alpha",},}- candidate := kccommon.Product{- ID: "kb/wiki/alpha",- DocID: "kb",- TenantID: "t1",- Variant: kccommon.VariantWiki,- Content: "# new markdown",- Vector: []float32{0, 1, 0, 0},- Meta: map[string]any{- "kind": "page",- "slug": "page/alpha",- "run_id": "run-9",- },- }-- groups := wikiMergeBatch(context.Background(), []MergeGroup{{- Existing: existing,- Candidates: []kccommon.Product{candidate},- }})-- merged := groups[0].Merged- if !strings.Contains(merged.Content, "# old markdown") || !strings.Contains(merged.Content, "# new markdown") {- t.Fatalf("content = %q, want both evidence blocks", merged.Content)- }- if !groups[0].Duplicate {- t.Fatalf("wiki candidate must be marked duplicate (replacement)")- }- if len(merged.Vector) != len(candidate.Vector) {- t.Fatalf("merged vector must be copied from the candidate, got %v", merged.Vector)- }- for i := range candidate.Vector {- if merged.Vector[i] != candidate.Vector[i] {- t.Fatalf("merged vector[%d] = %v, want candidate %v", i, merged.Vector, candidate.Vector)- }- }- // Identity and creation time come from the existing row; content-bearing and- // page metadata fields come from the candidate.- if merged.ID != existing.ID || merged.DocID != existing.DocID {- t.Fatalf("identity must be preserved: id=%q doc_id=%q", merged.ID, merged.DocID)- }- if merged.Meta["created_at"] != "2024-01-01" {- t.Fatalf("creation time must be preserved from the existing row, got %v", merged.Meta["created_at"])- }-}-func TestWikiEntityMergeRetainsBothEvidenceBodies(t *testing.T) {- existing := kccommon.Product{- ID: "page-1", DocID: "kb", Variant: kccommon.VariantWiki,- Content: "# Alpha\n\nold evidence",- Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "page_type": "entity", "source_doc_ids": []string{"doc-1"}},- }- incoming := kccommon.Product{- ID: "doc-page", DocID: "doc-2", Variant: kccommon.VariantWiki,- Content: "# Alpha\n\nnew evidence",- Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "page_type": "entity", "source_doc_ids": []string{"doc-2"}},+ consumer := &Consumer{}+ merged, err := consumer.mergeWikiProductsBySlug(t.Context(), "tenant-1", "kb-1", []kccommon.Product{page})+ if err != nil {+ t.Fatalf("mergeWikiProductsBySlug failed: %v", err)}- merged := wikiEntityMerge(existing, incoming)- if merged.Content == existing.Content || merged.Content == incoming.Content {- t.Fatalf("entity merge discarded evidence: %q", merged.Content)+ if len(merged) != 1 {+ t.Fatalf("merged pages = %d, want 1", len(merged))}- if len(metaStringSliceAny(merged.Meta, "source_doc_ids")) != 2 {- t.Fatalf("source docs = %#v, want both documents", merged.Meta["source_doc_ids"])- }- if merged.ID != existing.ID || merged.DocID != existing.DocID {- t.Fatalf("entity identity changed: %#v", merged)- }-}--func TestSplitMarkdownBlocksKeepsFencedCodeTogether(t *testing.T) {- blocks := splitMarkdownBlocks("before\n\n```go\nline one\n\nline two\n```\n\nafter")- if len(blocks) != 3 {- t.Fatalf("blocks=%d, want 3: %#v", len(blocks), blocks)+ if merged[0].Content != page.Content {+ t.Fatalf("single-source content changed: got %q, want %q", merged[0].Content, page.Content)}- if !strings.Contains(blocks[1], "line one\n\nline two") {- t.Fatalf("fenced block was split: %q", blocks[1])+ if merged[0].DocID != "kb-1" || !merged[0].Merged {+ t.Fatalf("dataset page identity not applied: doc_id=%q merged=%t", merged[0].DocID, merged[0].Merged)}}diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex ed2079fb15..caeb784d90 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -1089,7 +1089,7 @@ type wikiPageProjection struct {// re-materializes the page graph. See the Writer interface doc for the// delete-then-insert (non-atomic) contract.func (w engineWriter) ProjectWikiGraph(ctx context.Context, tenant, kb string) error {- pages, err := w.loadMergedWikiPages(ctx, tenant, kb)+ pages, err := w.loadActiveDocumentWikiPages(ctx, tenant, kb)if err != nil {return err}@@ -1116,6 +1116,76 @@ func (w engineWriter) ProjectWikiGraph(ctx context.Context, tenant, kb string) ereturn w.insertWikiGraphChunks(ctx, tenant, kb, rows)}+// loadActiveDocumentWikiPages loads the immutable document-level page+// contributions that are currently enabled and folds equal slugs without+// touching their content. Graph projection must use these rows instead of the+// LLM-composed dataset page, otherwise disabling one of several contributors+// would leave that contributor's entities and edges in the graph.+func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, kb string) ([]wikiPageProjection, error) {+ eng := w.eng+ if eng == nil {+ eng = engine.Get()+ }+ if eng == nil {+ return nil, nil+ }+ const batchSize = 2000+ bySlug := make(map[string]wikiPageProjection)+ for offset := 0; ; offset += batchSize {+ result, err := eng.Search(ctx, &types.SearchRequest{+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)},+ KbIDs: []string{kb},+ Filter: map[string]interface{}{+ "compile_kwd": compileKwdWikiPage, "available_int": 0,+ "scope_kwd": "doc", "kb_id": kb,+ },+ SelectFields: []string{+ "slug_kwd", "page_type_kwd", "title_kwd", "entity_names_kwd",+ "summary_with_weight", "outlinks_kwd", "source_doc_ids", "source_chunk_ids",+ },+ Limit: batchSize, Offset: offset,+ OrderBy: (&types.OrderByExpr{}).Asc("slug_kwd").Asc("doc_id"),+ })+ if err != nil {+ return nil, err+ }+ if result == nil || len(result.Chunks) == 0 {+ break+ }+ for _, row := range result.Chunks {+ slug := metaString(row, "slug_kwd")+ if slug == "" {+ continue+ }+ page := bySlug[slug]+ if page.Slug == "" {+ page.Slug = slug+ page.PageType = metaString(row, "page_type_kwd")+ page.Title = metaString(row, "title_kwd")+ page.Summary = metaString(row, "summary_with_weight")+ }+ page.Aliases = unionStrs(page.Aliases, metaStringSlice(row, "entity_names_kwd"))+ page.Outlinks = unionStrs(page.Outlinks, metaStringSlice(row, "outlinks_kwd"))+ page.SourceDocIDs = unionStrs(page.SourceDocIDs, metaStringSlice(row, "source_doc_ids"))+ page.SourceChunkIDs = unionStrs(page.SourceChunkIDs, metaStringSlice(row, "source_chunk_ids"))+ bySlug[slug] = page+ }+ if len(result.Chunks) < batchSize {+ break+ }+ }+ slugs := make([]string, 0, len(bySlug))+ for slug := range bySlug {+ slugs = append(slugs, slug)+ }+ sort.Strings(slugs)+ pages := make([]wikiPageProjection, 0, len(slugs))+ for _, slug := range slugs {+ pages = append(pages, bySlug[slug])+ }+ return pages, nil+}+// DropWikiGraph deletes every wiki_entity / wiki_relation row for the dataset.func (w engineWriter) DropWikiGraph(ctx context.Context, tenant, kb string) error {return w.dropWikiGraph(ctx, tenant, kb)diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.goindex 8f50a92f69..67353c0ef1 100644--- a/internal/ingestion/pipeline/pipeline.go+++ b/internal/ingestion/pipeline/pipeline.go@@ -467,6 +467,7 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParam// runs), in which case TrackProgress is a no-op — progress is an// observability concern, not a data dependency.runCtx = runtime.WithProgressCallback(runCtx, p.componentProgressCallback(ctx))+ runCtx = runtime.WithProgressMessageCallback(runCtx, p.componentProgressMessageCallback(ctx))current := cloneMapOrEmpty(inputs)@@ -712,3 +713,22 @@ func (p *Pipeline) componentProgressCallback(ctx context.Context) runtime.Progre})}}++type detailedProgressSink interface {+ OnComponentMessage(ctx context.Context, taskID, documentID, component, message string)+}++func (p *Pipeline) componentProgressMessageCallback(ctx context.Context) runtime.ProgressMessageCallback {+ sink, ok := p.sink.(detailedProgressSink)+ if !ok {+ return nil+ }+ return func(component, message string) {+ common.Info("component progress detail",+ zap.String("component", component),+ zap.String("task_id", p.taskID),+ zap.String("document_id", p.documentID),+ zap.String("message", message))+ sink.OnComponentMessage(ctx, p.taskID, p.documentID, component, message)+ }+}diff --git a/internal/ingestion/service/progress_sink.go b/internal/ingestion/service/progress_sink.goindex efb5469e3a..c7f5a826ec 100644--- a/internal/ingestion/service/progress_sink.go+++ b/internal/ingestion/service/progress_sink.go@@ -114,6 +114,38 @@ func (s *progressSink) OnComponentProgress(ctx context.Context, ev pipeline.Prog}}+// OnComponentMessage appends detailed compiler-stage information without+// creating a lifecycle row. This keeps component_total and completion percent+// based on actual canvas components while exposing MAP/REDUCE/PLAN/REFINE+// counts through the existing document progress log.+func (s *progressSink) OnComponentMessage(ctx context.Context, taskID, docID, component, message string) {+ if docID == "" || message == "" {+ return+ }+ s.logMu.Lock()+ defer s.logMu.Unlock()+ doc, err := s.docSvc.GetDocumentByID(ctx, docID)+ if err != nil {+ common.Error(fmt.Sprintf("progressSink: read document %s for detail failed", docID), err)+ return+ }+ if doc == nil {+ return+ }+ run := ""+ if doc.Run != nil {+ run = *doc.Run+ }+ log, err := s.accumulateLogLocked(ctx, docID, fmt.Sprintf("%s: %s", component, message))+ if err != nil {+ common.Error(fmt.Sprintf("progressSink: append detail for document %s failed", docID), err)+ return+ }+ if err := s.docSvc.UpdateRunProgress(ctx, docID, doc.Progress, run, log); err != nil {+ common.Error(fmt.Sprintf("progressSink: persist detail for document %s failed", docID), err)+ }+}+// updateDocumentProgress serializes log accumulation with the corresponding// document write. Without holding the same lock across both operations, a// slower database write can store an older snapshot after a newer event hasdiff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex 1c2f66f255..10127128be 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -28,7 +28,7 @@ import (enginetypes "ragflow/internal/engine/types""ragflow/internal/entity""ragflow/internal/entity/models"- _ "ragflow/internal/ingestion/component/knowledge_compiler"+ knowledgecompiler "ragflow/internal/ingestion/component/knowledge_compiler"kc "ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/ingestion/knowledge_compile""ragflow/internal/service"@@ -54,6 +54,402 @@ func init() {kc.SetDepsResolver(newKnowledgeCompilerDepsResolver())kc.SetGroupResolver(newKnowledgeCompilerGroupResolver())kc.SetTemplateResolver(newKnowledgeCompilerTemplateResolver())+ knowledge_compile.SetWikiDirtyCompiler(compileDirtyWikiDocument)+}++func compileDirtyWikiDocument(ctx context.Context, request knowledge_compile.WikiDirtyRequest) error {+ doc, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, request.DocumentID)+ if err != nil {+ if err == gorm.ErrRecordNotFound {+ return nil+ }+ return fmt.Errorf("Wiki dirty compile: load document: %w", err)+ }+ if doc.KbID != request.DatasetID || (doc.Status != nil && *doc.Status == "0") {+ return replaceDirtyWikiProducts(ctx, request, nil, nil, nil, nil, true)+ }+ compilerParams, err := loadWikiCompilerParams(ctx, doc)+ if err != nil {+ return err+ }+ templateIDs, err := resolveWikiTemplateIDs(ctx, request.TenantID, compilerParams)+ if err != nil {+ return err+ }+ if len(templateIDs) == 0 {+ return replaceDirtyWikiProducts(ctx, request, nil, nil, nil, nil, true)+ }+ sourceChunks, err := loadActiveSourceChunks(ctx, request)+ if err != nil {+ return err+ }+ if len(sourceChunks) == 0 {+ return replaceDirtyWikiProducts(ctx, request, nil, nil, nil, nil, true)+ }+ compiled := make([]map[string]any, 0)+ affectedSlugs := make([]string, 0)+ removedSlugs := make([]string, 0)+ activeStates := make([]kc.WikiMapActiveState, 0)+ for _, templateID := range templateIDs {+ params := copyStringAnyMap(compilerParams)+ delete(params, "compilation_template_group_id")+ delete(params, "compilation_template_group_ids")+ params["compilation_template_id"] = templateID+ component, err := knowledgecompiler.NewKnowledgeCompilerComponent("Compiler", params)+ if err != nil {+ return err+ }+ output, err := component.Invoke(ctx, dao.DB, map[string]any{+ "chunks": sourceChunks,+ "tenant_id": request.TenantID,+ "dataset_id": request.DatasetID,+ "kb_id": request.DatasetID,+ "doc_id": request.DocumentID,+ "wiki_incremental": true,+ })+ if err != nil {+ return fmt.Errorf("Wiki dirty compile document %s: %w", request.DocumentID, err)+ }+ compiled = append(compiled, wikiCompiledRows(output)...)+ affectedSlugs = append(affectedSlugs, stringValues(output["wiki_affected_slugs"])...)+ removedSlugs = append(removedSlugs, stringValues(output["wiki_removed_slugs"])...)+ states, err := wikiActiveStates(output)+ if err != nil {+ return fmt.Errorf("Wiki dirty compile document %s: %w", request.DocumentID, err)+ }+ activeStates = append(activeStates, states...)+ }+ if len(affectedSlugs) == 0 && len(removedSlugs) == 0 {+ return persistDirtyWikiActiveStates(ctx, request, activeStates)+ }+ return replaceDirtyWikiProducts(ctx, request, compiled, uniqueStrings(affectedSlugs), uniqueStrings(removedSlugs), activeStates, false)+}++func loadWikiCompilerParams(ctx context.Context, doc *entity.Document) (map[string]any, error) {+ if params := findCompilerParams(map[string]any(doc.ParserConfig)); params != nil {+ return params, nil+ }+ if doc.PipelineID == nil || strings.TrimSpace(*doc.PipelineID) == "" {+ return map[string]any{}, nil+ }+ canvas, err := dao.NewUserCanvasDAO().GetByID(ctx, dao.DB, *doc.PipelineID)+ if err != nil {+ if err == dao.ErrUserCanvasNotFound {+ return map[string]any{}, nil+ }+ return nil, err+ }+ if params := findCompilerParams(map[string]any(canvas.DSL)); params != nil {+ return params, nil+ }+ return map[string]any{}, nil+}++func findCompilerParams(value any) map[string]any {+ switch typed := value.(type) {+ case entity.JSONMap:+ return findCompilerParams(map[string]any(typed))+ case map[string]any:+ if _, hasGroup := typed["compilation_template_group_id"]; hasGroup {+ return copyStringAnyMap(typed)+ }+ if _, hasGroups := typed["compilation_template_group_ids"]; hasGroups {+ return copyStringAnyMap(typed)+ }+ if _, hasTemplate := typed["compilation_template_id"]; hasTemplate {+ return copyStringAnyMap(typed)+ }+ for _, child := range typed {+ if params := findCompilerParams(child); params != nil {+ return params+ }+ }+ case []any:+ for _, child := range typed {+ if params := findCompilerParams(child); params != nil {+ return params+ }+ }+ }+ return nil+}++func resolveWikiTemplateIDs(ctx context.Context, tenantID string, params map[string]any) ([]string, error) {+ ids := make([]string, 0)+ if templateID, ok := params["compilation_template_id"].(string); ok && strings.TrimSpace(templateID) != "" {+ ids = append(ids, strings.TrimSpace(templateID))+ }+ groupIDs := stringValues(params["compilation_template_group_id"])+ groupIDs = append(groupIDs, stringValues(params["compilation_template_group_ids"])...)+ if len(groupIDs) > 0 {+ resolved, err := dao.NewCompilationTemplateDAO().ResolveGroupTemplateIDs(ctx, dao.DB, tenantID, uniqueStrings(groupIDs))+ if err != nil {+ return nil, err+ }+ ids = append(ids, resolved...)+ }+ ids = uniqueStrings(ids)+ if len(ids) == 0 {+ return nil, nil+ }+ var templates []entity.CompilationTemplate+ if err := dao.DB.WithContext(ctx).+ Where("id IN ? AND status = ?", ids, string(entity.StatusValid)).Find(&templates).Error; err != nil {+ return nil, err+ }+ wikiIDs := make([]string, 0, len(templates))+ for _, template := range templates {+ kind := template.Kind+ if configured, ok := template.Config["kind"].(string); ok && strings.TrimSpace(configured) != "" {+ kind = configured+ }+ if strings.EqualFold(strings.TrimSpace(kind), "wiki") || strings.EqualFold(strings.TrimSpace(kind), "wiki_page") {+ wikiIDs = append(wikiIDs, template.ID)+ }+ }+ return uniqueStrings(wikiIDs), nil+}++func loadActiveSourceChunks(ctx context.Context, request knowledge_compile.WikiDirtyRequest) ([]map[string]any, error) {+ docEngine := engine.Get()+ if docEngine == nil {+ return nil, fmt.Errorf("Wiki dirty compile: document engine is unavailable")+ }+ chunks := make([]map[string]any, 0)+ for offset := 0; ; offset += 1000 {+ result, err := docEngine.Search(ctx, &enginetypes.SearchRequest{+ IndexNames: []string{fmt.Sprintf("ragflow_%s", request.TenantID)},+ KbIDs: []string{request.DatasetID},+ Offset: offset,+ Limit: 1000,+ SelectFields: []string{+ "id", "doc_id", "docnm_kwd", "content_with_weight", "available_int", "compile_kwd",+ },+ Filter: map[string]any{+ "doc_id": []string{request.DocumentID},+ "available_int": 1,+ },+ })+ if err != nil {+ return nil, err+ }+ if result == nil {+ break+ }+ for _, row := range result.Chunks {+ if strings.TrimSpace(anyString(row["compile_kwd"])) != "" {+ continue+ }+ chunks = append(chunks, row)+ }+ if len(result.Chunks) == 0 || int64(offset+len(result.Chunks)) >= result.Total {+ break+ }+ }+ return chunks, nil+}++func wikiCompiledRows(output map[string]any) []map[string]any {+ rows := make([]map[string]any, 0)+ items, _ := output["chunks"].([]any)+ for _, item := range items {+ row, ok := item.(map[string]any)+ if !ok {+ continue+ }+ compileKWD := strings.TrimSpace(anyString(row["compile_kwd"]))+ if compileKWD != "wiki_page" && compileKWD != "wiki_section" {+ continue+ }+ row["available_int"] = 0+ rows = append(rows, row)+ }+ return rows+}++func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.WikiDirtyRequest, rows []map[string]any, affectedSlugs, removedSlugs []string, activeStates []kc.WikiMapActiveState, fullReplace bool) error {+ var dirty entity.WikiDocumentDirty+ if err := dao.DB.WithContext(ctx).Where("document_id = ?", request.DocumentID).First(&dirty).Error; err != nil {+ if err == gorm.ErrRecordNotFound {+ return nil+ }+ return err+ }+ if dirty.Revision != request.Revision {+ return nil+ }+ docEngine := engine.Get()+ if docEngine == nil {+ return fmt.Errorf("Wiki dirty compile: document engine is unavailable")+ }+ indexName := fmt.Sprintf("ragflow_%s", request.TenantID)+ existing, err := docEngine.Search(ctx, &enginetypes.SearchRequest{+ IndexNames: []string{indexName},+ KbIDs: []string{request.DatasetID},+ Limit: 10000,+ SelectFields: []string{"id", "compile_kwd", "slug_kwd", "parent_id"},+ Filter: map[string]any{+ "doc_id": []string{request.DocumentID},+ "compile_kwd": []string{"wiki_page", "wiki_section"},+ "available_int": 0,+ },+ })+ if err != nil {+ return err+ }+ newIDs := make(map[string]struct{}, len(rows))+ for _, row := range rows {+ if id := anyString(row["id"]); id != "" {+ newIDs[id] = struct{}{}+ }+ }+ if len(rows) > 0 {+ if _, err := docEngine.InsertChunks(ctx, rows, indexName, request.DatasetID); err != nil {+ return err+ }+ }+ affected := make(map[string]struct{}, len(affectedSlugs)+len(removedSlugs))+ for _, slug := range append(append([]string(nil), affectedSlugs...), removedSlugs...) {+ affected[slug] = struct{}{}+ }+ affectedPageIDs := make(map[string]struct{})+ if existing != nil && !fullReplace {+ for _, row := range existing.Chunks {+ if anyString(row["compile_kwd"]) != "wiki_page" {+ continue+ }+ if _, ok := affected[anyString(row["slug_kwd"])]; ok {+ affectedPageIDs[anyString(row["id"])] = struct{}{}+ }+ }+ }+ if existing != nil && fullReplace {+ for _, row := range existing.Chunks {+ if anyString(row["compile_kwd"]) == "wiki_page" {+ removedSlugs = append(removedSlugs, anyString(row["slug_kwd"]))+ }+ }+ removedSlugs = uniqueStrings(removedSlugs)+ }+ staleIDs := make([]string, 0)+ if existing != nil {+ for _, row := range existing.Chunks {+ id := anyString(row["id"])+ _, pageAffected := affectedPageIDs[id]+ _, sectionAffected := affectedPageIDs[anyString(row["parent_id"])]+ inScope := fullReplace || pageAffected || sectionAffected+ if _, keep := newIDs[id]; id != "" && inScope && !keep {+ staleIDs = append(staleIDs, id)+ }+ }+ }+ if len(staleIDs) > 0 {+ if _, err := docEngine.DeleteChunks(ctx, map[string]any{"id": staleIDs, "kb_id": request.DatasetID}, indexName, request.DatasetID); err != nil {+ return err+ }+ }+ if fullReplace {+ if err := clearWikiActiveStates(ctx, docEngine, request); err != nil {+ return err+ }+ }+ if err := putWikiActiveStates(ctx, docEngine, activeStates); err != nil {+ return err+ }+ return knowledge_compile.PublishCompleted(ctx, request.TenantID, request.DatasetID, request.DocumentID, []string{"wiki"})+}++func clearWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, request knowledge_compile.WikiDirtyRequest) error {+ _, err := docEngine.DeleteChunks(ctx, map[string]any{+ "kb_id": request.DatasetID,+ "compile_kwd": "wiki_map_active",+ "available_int": 0,+ "source_doc_ids": []string{request.DocumentID},+ }, fmt.Sprintf("ragflow_%s", request.TenantID), request.DatasetID)+ if err != nil {+ return fmt.Errorf("clear Wiki active MAP state for document %s: %w", request.DocumentID, err)+ }+ return nil+}++func persistDirtyWikiActiveStates(ctx context.Context, request knowledge_compile.WikiDirtyRequest, states []kc.WikiMapActiveState) error {+ if len(states) == 0 {+ return nil+ }+ var dirty entity.WikiDocumentDirty+ if err := dao.DB.WithContext(ctx).Where("document_id = ?", request.DocumentID).First(&dirty).Error; err != nil {+ if err == gorm.ErrRecordNotFound {+ return nil+ }+ return err+ }+ if dirty.Revision != request.Revision {+ return nil+ }+ docEngine := engine.Get()+ if docEngine == nil {+ return fmt.Errorf("Wiki dirty compile: document engine is unavailable")+ }+ return putWikiActiveStates(ctx, docEngine, states)+}++func putWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, states []kc.WikiMapActiveState) error {+ if len(states) == 0 {+ return nil+ }+ store, ok := knowledge_compile.NewWikiMapVersionStore(docEngine).(kc.WikiMapActiveStateStore)+ if !ok {+ return fmt.Errorf("Wiki dirty compile: active MAP store is unavailable")+ }+ for _, state := range states {+ if err := store.PutWikiMapActiveState(ctx, state); err != nil {+ return err+ }+ }+ return nil+}++func copyStringAnyMap(source map[string]any) map[string]any {+ result := make(map[string]any, len(source))+ for key, value := range source {+ result[key] = value+ }+ return result+}++func stringValues(value any) []string {+ switch typed := value.(type) {+ case string:+ return []string{typed}+ case []string:+ return typed+ case []any:+ result := make([]string, 0, len(typed))+ for _, item := range typed {+ if value, ok := item.(string); ok {+ result = append(result, value)+ }+ }+ return result+ }+ return nil+}++func uniqueStrings(values []string) []string {+ seen := make(map[string]struct{}, len(values))+ result := make([]string, 0, len(values))+ for _, value := range values {+ value = strings.TrimSpace(value)+ if value == "" {+ continue+ }+ if _, ok := seen[value]; ok {+ continue+ }+ seen[value] = struct{}{}+ result = append(result, value)+ }+ return result}// newKnowledgeCompilerGroupResolver builds the production GroupResolver backed bydiff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 40a20cebe5..ad9a480c33 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -28,6 +28,7 @@ import ("ragflow/internal/common""ragflow/internal/dao""ragflow/internal/engine"+ enginetypes "ragflow/internal/engine/types""ragflow/internal/entity""ragflow/internal/ingestion/component"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"@@ -261,9 +262,23 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map// Ordinary source chunks stay available_int=1 (the index default).markCompiledProductsHidden(chunks)+ oldCompiledProductIDs, err := s.loadDocumentCompiledProductIDs(ctx)+ if err != nil {+ return nil, err+ }if err := s.indexWriter.Write(ctx, chunks); err != nil {return nil, err}+ if err := s.reconcileDocumentCompiledProducts(ctx, oldCompiledProductIDs, chunks); err != nil {+ return nil, err+ }+ activeStates, err := wikiActiveStates(pipelineOutput)+ if err != nil {+ return nil, err+ }+ if err := putWikiActiveStates(ctx, engine.Get(), activeStates); err != nil {+ return nil, fmt.Errorf("persist Wiki active MAP state: %w", err)+ }// All chunks are now persisted. Notify the dataset-level post-processing consumer// (§11) that this document is complete: its compiled products were written@@ -407,6 +422,86 @@ func markCompiledProductsHidden(chunks []map[string]any) {}}+// loadDocumentCompiledProductIDs snapshots the previous successful document+// compiler generation before the new pipeline output is written. Reading first+// avoids relying on immediate search visibility after a bulk index write.+func (s *PipelineExecutor) loadDocumentCompiledProductIDs(ctx context.Context) ([]string, error) {+ docEngine := engine.Get()+ if docEngine == nil || s == nil || s.taskCtx == nil {+ return nil, nil+ }+ const pageSize = 1000+ indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)+ oldIDs := make([]string, 0)+ for offset := 0; ; offset += pageSize {+ result, err := docEngine.Search(ctx, &enginetypes.SearchRequest{+ IndexNames: []string{indexName},+ KbIDs: []string{s.taskCtx.Doc.KbID},+ Offset: offset,+ Limit: pageSize,+ SelectFields: []string{"id", "compile_kwd"},+ Filter: map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}},+ })+ if err != nil {+ return nil, fmt.Errorf("load document compiler products: %w", err)+ }+ if result == nil || len(result.Chunks) == 0 {+ break+ }+ for _, row := range result.Chunks {+ if strings.TrimSpace(asCompiledKwd(row)) == "" {+ continue+ }+ if id := strings.TrimSpace(anyString(row["id"])); id != "" {+ oldIDs = append(oldIDs, id)+ }+ }+ if int64(offset+len(result.Chunks)) >= result.Total {+ break+ }+ }+ return oldIDs, nil+}++// reconcileDocumentCompiledProducts advances the document-level compiler+// generation only after the new pipeline output has been persisted. Reparse+// preparation keeps the previous compiled rows so a failed run does not erase+// the last usable document Wiki; this method removes rows absent from the new+// successful generation immediately before the dataset completion event.+func (s *PipelineExecutor) reconcileDocumentCompiledProducts(ctx context.Context, oldIDs []string, chunks []map[string]any) error {+ docEngine := engine.Get()+ if docEngine == nil || s == nil || s.taskCtx == nil {+ return nil+ }+ newIDs := make(map[string]struct{})+ for _, chunk := range chunks {+ if strings.TrimSpace(asCompiledKwd(chunk)) == "" {+ continue+ }+ if id := strings.TrimSpace(anyString(chunk["id"])); id != "" {+ newIDs[id] = struct{}{}+ }+ }+ staleIDs := make([]string, 0, len(oldIDs))+ for _, id := range oldIDs {+ if _, keep := newIDs[id]; !keep {+ staleIDs = append(staleIDs, id)+ }+ }+ const pageSize = 1000+ indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)+ for start := 0; start < len(staleIDs); start += pageSize {+ end := min(start+pageSize, len(staleIDs))+ if _, err := docEngine.DeleteChunks(ctx, map[string]any{+ "id": staleIDs[start:end],+ "kb_id": s.taskCtx.Doc.KbID,+ }, indexName, s.taskCtx.Doc.KbID); err != nil {+ return fmt.Errorf("delete stale document compiler products: %w", err)+ }+ }+ return nil+}+// compiledVariants returns the sorted, de-duplicated set of compile types a// document's compiled products carry. It reads the authoritative// `compilation_template_kind_kwd` the KnowledgeCompiler component stamps on each@@ -475,6 +570,63 @@ func asCompiledKwd(c map[string]any) string {return ""}+func wikiActiveStates(output map[string]any) ([]kccommon.WikiMapActiveState, error) {+ if output == nil || output["wiki_active_map_states"] == nil {+ return nil, nil+ }+ appendState := func(states []kccommon.WikiMapActiveState, value map[string]any) ([]kccommon.WikiMapActiveState, error) {+ state := kccommon.WikiMapActiveState{+ Key: strings.TrimSpace(anyString(value["key"])),+ TenantID: strings.TrimSpace(anyString(value["tenant_id"])),+ DatasetID: strings.TrimSpace(anyString(value["dataset_id"])),+ DocumentID: strings.TrimSpace(anyString(value["document_id"])),+ }+ switch payload := value["payload"].(type) {+ case string:+ state.Payload = []byte(payload)+ case []byte:+ state.Payload = append([]byte(nil), payload...)+ default:+ return nil, fmt.Errorf("decode Wiki active MAP state: unsupported payload type %T", value["payload"])+ }+ if state.Key == "" || state.TenantID == "" || state.DatasetID == "" || state.DocumentID == "" {+ return nil, fmt.Errorf("decode Wiki active MAP state: incomplete scope")+ }+ return append(states, state), nil+ }++ switch values := output["wiki_active_map_states"].(type) {+ case []kccommon.WikiMapActiveState:+ return append([]kccommon.WikiMapActiveState(nil), values...), nil+ case []map[string]any:+ states := make([]kccommon.WikiMapActiveState, 0, len(values))+ for _, value := range values {+ var err error+ states, err = appendState(states, value)+ if err != nil {+ return nil, err+ }+ }+ return states, nil+ case []any:+ states := make([]kccommon.WikiMapActiveState, 0, len(values))+ for _, raw := range values {+ value, ok := raw.(map[string]any)+ if !ok {+ return nil, fmt.Errorf("decode Wiki active MAP state: unsupported item type %T", raw)+ }+ var err error+ states, err = appendState(states, value)+ if err != nil {+ return nil, err+ }+ }+ return states, nil+ default:+ return nil, fmt.Errorf("decode Wiki active MAP states: unsupported type %T", output["wiki_active_map_states"])+ }+}+func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string) {var dslMap entity.JSONMapif err := json.Unmarshal([]byte(dsl), &dslMap); err != nil {diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex 80222d12f6..021d409fa9 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -51,6 +51,20 @@ func TestMarkCompiledProductsHidden(t *testing.T) {}}+func TestWikiActiveStatesDecodeCheckpointValues(t *testing.T) {+ states, err := wikiActiveStates(map[string]any{+ "wiki_active_map_states": []any{map[string]any{+ "key": "state-1", "tenant_id": "tenant-1", "dataset_id": "kb-1", "document_id": "doc-1", "payload": `{"plan":[]}`,+ }},+ })+ if err != nil {+ t.Fatalf("wikiActiveStates failed: %v", err)+ }+ if len(states) != 1 || states[0].Key != "state-1" || string(states[0].Payload) != `{"plan":[]}` {+ t.Fatalf("decoded states = %#v", states)+ }+}+func makeTaskCtx() *TaskContext {return &TaskContext{IngestionTask: &entity.IngestionTask{diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex 23a34f8f39..07567891cb 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -42,6 +42,7 @@ import ("ragflow/internal/dao""ragflow/internal/engine""ragflow/internal/engine/types"+ "ragflow/internal/ingestion/knowledge_compile""ragflow/internal/service""ragflow/internal/service/document""ragflow/internal/service/nlp"@@ -1061,6 +1062,7 @@ func (s *ChunkService) SwitchChunks(ctx context.Context, userID, datasetID, docureturn err}}+ s.markWikiDirty(ctx, targetTenantID, datasetID, documentID, chunkIDs)return nil}@@ -1201,6 +1203,9 @@ func (s *ChunkService) UpdateChunk(ctx context.Context, req *service.UpdateChunkif err != nil {return fmt.Errorf("failed to update chunk: %w", err)}+ if req.Content != nil || req.Available != nil {+ s.markWikiDirty(ctx, targetTenantID, req.DatasetID, req.DocumentID, []string{req.ChunkID})+ }return nil}@@ -1273,6 +1278,7 @@ func (s *ChunkService) RemoveChunks(ctx context.Context, req *service.RemoveChunif err = s.decrementChunkStats(req.DocID, doc.KbID, 0, deletedCount, 0); err != nil {return deletedCount, fmt.Errorf("failed to update chunk stats: %w", err)}+ s.markWikiDirty(ctx, targetTenantID, doc.KbID, req.DocID, req.ChunkIDs)}return deletedCount, nil@@ -1408,6 +1414,7 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkRequesif err = s.incrementChunkStats(req.DocumentID, req.DatasetID, tokenNum, 1, 0); err != nil {return nil, addChunkError{code: common.CodeServerError, message: fmt.Sprintf("increment chunk stats: %v", err)}}+ s.markWikiDirty(ctx, kb.TenantID, req.DatasetID, req.DocumentID, []string{chunkID})renamedChunk := map[string]interface{}{"id": chunkID,@@ -1430,6 +1437,15 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkRequesreturn &service.AddChunkResponse{Chunk: renamedChunk}, nil}+func (s *ChunkService) markWikiDirty(ctx context.Context, tenantID, datasetID, documentID string, chunkIDs []string) {+ markCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)+ defer cancel()+ if err := knowledge_compile.MarkWikiDocumentDirty(markCtx, tenantID, datasetID, documentID, chunkIDs); err != nil {+ common.Warn("chunk mutation: failed to schedule Wiki refresh",+ zap.String("document_id", documentID), zap.Error(err))+ }+}+type addChunkError struct {code common.ErrorCodemessage stringdiff --git a/internal/service/dataset/compilation_status.go b/internal/service/dataset/compilation_status.goindex 0c996d565b..712ab4cc1d 100644--- a/internal/service/dataset/compilation_status.go+++ b/internal/service/dataset/compilation_status.go@@ -25,6 +25,9 @@ import (type CompilationStatus struct {State string `json:"state"` // idle | pending | running | completedError string `json:"error,omitempty"` // most recent batch diagnostic (empty when none)+ Progress float64 `json:"progress"` // current claimed batch progress+ CurrentPhase string `json:"current_phase,omitempty"` // current compiler phase+ ProgressMsg string `json:"progress_msg,omitempty"` // bounded phase log for the UIInflight int `json:"inflight"` // entries currently claimed (in-flight batch)Backlog int `json:"backlog"` // entries still waiting to be claimedLastCompletedAt *time.Time `json:"last_completed_at,omitempty"` // last backlog drain@@ -61,6 +64,9 @@ func (d *DatasetService) GetDatasetCompilationStatus(ctx context.Context, userIDst.State = entity.DatasetStateIdle}st.Error = row.ErrorMsg+ st.Progress = row.Progress+ st.CurrentPhase = row.CurrentPhase+ st.ProgressMsg = row.ProgressMsgst.Inflight = jsonArrayLen(row.InflightDocIDs)st.Backlog = jsonArrayLen(row.BacklogDocIDs)st.LastCompletedAt = row.LastCompletedAtdiff --git a/internal/service/dataset/compilation_status_test.go b/internal/service/dataset/compilation_status_test.goindex 9b910fbc4b..8753061e48 100644--- a/internal/service/dataset/compilation_status_test.go+++ b/internal/service/dataset/compilation_status_test.go@@ -129,6 +129,9 @@ func TestGetDatasetCompilationStatus_FullOutput(t *testing.T) {InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","seq":1},{"doc_id":"d2","event_type":"completed","seq":2}]`,State: entity.DatasetStateRunning,ErrorMsg: "merge failed: boom",+ Progress: 0.72,+ CurrentPhase: "routing_pages",+ ProgressMsg: "Routing 55 Wiki pages",LastCompletedAt: &lastDone,}if err := db.Create(&row).Error; err != nil {@@ -152,6 +155,9 @@ func TestGetDatasetCompilationStatus_FullOutput(t *testing.T) {if st.Error != "merge failed: boom" {t.Fatalf("error=%q want %q", st.Error, "merge failed: boom")}+ if st.Progress != 0.72 || st.CurrentPhase != "routing_pages" || st.ProgressMsg != "Routing 55 Wiki pages" {+ t.Fatalf("progress fields not mapped: progress=%v phase=%q msg=%q", st.Progress, st.CurrentPhase, st.ProgressMsg)+ }if st.LastCompletedAt == nil || !st.LastCompletedAt.Equal(lastDone) {t.Fatalf("last_completed_at=%v want %v", st.LastCompletedAt, lastDone)}diff --git a/internal/service/dataset/ingestion.go b/internal/service/dataset/ingestion.goindex 84750a8c47..75ee64fdaf 100644--- a/internal/service/dataset/ingestion.go+++ b/internal/service/dataset/ingestion.go@@ -123,8 +123,15 @@ func datasetIngestionLogToMap(log *entity.PipelineOperationLog) map[string]inter"task_type": log.TaskType,"operation_status": log.OperationStatus,"progress": log.Progress,+ "process_begin_at": log.ProcessBeginAt,+ "process_duration": log.ProcessDuration,+ "progress_msg": log.ProgressMsg,+ "dsl": log.DSL,+ "avatar": log.Avatar,"create_time": log.CreateTime,+ "create_date": log.CreateDate,"update_time": log.UpdateTime,+ "update_date": log.UpdateDate,}if log.PipelineID != nil {m["pipeline_id"] = *log.PipelineIDdiff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex 37ad7cd953..d374bf18e9 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -23,8 +23,11 @@ import ("golang.org/x/text/collate""golang.org/x/text/language"+ "gorm.io/gorm"+ "ragflow/internal/dao""ragflow/internal/engine""ragflow/internal/engine/types"+ "ragflow/internal/entity""ragflow/internal/service/nav")@@ -480,33 +483,206 @@ type WikiAlteration struct {// GetWikiAlteration returns the wiki alteration summary for a dataset.func (s *DatasetArtifactService) GetWikiAlteration(ctx context.Context, tenantID, datasetID string) (*WikiAlteration, error) {- chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}},- []string{"source_doc_ids"}, 0, 10000, nil)- if err != nil {- return nil, err- }involved := map[string]struct{}{}- for _, c := range chunks {- for _, d := range toStringSlice(c["source_doc_ids"]) {- involved[d] = struct{}{}+ for offset := 0; ; offset += 1000 {+ chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID,+ map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}},+ []string{"source_doc_ids"}, offset, 1000, nil)+ if err != nil {+ return nil, err+ }+ for _, c := range chunks {+ for _, d := range toStringSlice(c["source_doc_ids"]) {+ if d != "" {+ involved[d] = struct{}{}+ }+ }+ }+ if len(chunks) == 0 || int64(offset+len(chunks)) >= total {+ break}}- ids := make([]string, 0, len(involved))- for d := range involved {- ids = append(ids, d)+ // The database is the source of truth for the current document set. The+ // previous implementation returned the source_doc_ids from the compiled+ // pages as both sides of the comparison, which made deletion impossible to+ // observe. In particular, a deleted document remains in source_doc_ids until+ // the dataset-level consumer removes the old page.+ var documents []entity.Document+ if err := dao.DB.WithContext(ctx).Where("kb_id = ?", datasetID).Find(&documents).Error; err != nil {+ return nil, fmt.Errorf("list dataset documents for wiki alteration: %w", err)+ }+ eligible := make(map[string]struct{}, len(documents))+ for i := range documents {+ if documents[i].Status != nil && *documents[i].Status == "0" {+ continue+ }+ pipelineID := ""+ if documents[i].PipelineID != nil {+ pipelineID = *documents[i].PipelineID+ }+ ok, err := s.documentHasWikiTemplate(ctx, tenantID, documents[i].ParserConfig, pipelineID)+ if err != nil {+ return nil, err+ }+ if ok {+ eligible[documents[i].ID] = struct{}{}+ }}- sort.Strings(ids)++ removedIDs := setDifference(involved, eligible)+ newlyUploadedIDs := setDifference(eligible, involved)return &WikiAlteration{- Removed: 0,- NewlyUploaded: 0,- RemovedDocIDs: []string{},- NewlyUploadedDocIDs: []string{},- InvolvedDocIDs: ids,- EligibleDocIDs: ids,+ Removed: len(removedIDs),+ NewlyUploaded: len(newlyUploadedIDs),+ RemovedDocIDs: removedIDs,+ NewlyUploadedDocIDs: newlyUploadedIDs,+ InvolvedDocIDs: sortedSetKeys(involved),+ EligibleDocIDs: sortedSetKeys(eligible),}, nil}+// documentHasWikiTemplate reports whether a document's saved pipeline config+// contains a valid wiki compiler template. A document is eligible only when it+// is configured for wiki compilation; treating every document in the dataset+// as eligible would hide both template removal and document deletion.+func (s *DatasetArtifactService) documentHasWikiTemplate(ctx context.Context, tenantID string, config entity.JSONMap, pipelineID string) (bool, error) {+ ok, err := s.valueHasWikiTemplate(ctx, tenantID, config)+ if err != nil || ok {+ return ok, err+ }+ if pipelineID == "" {+ return false, nil+ }+ var canvas entity.UserCanvas+ if err := dao.DB.WithContext(ctx).Where("id = ?", pipelineID).First(&canvas).Error; err != nil {+ if err == gorm.ErrRecordNotFound {+ return false, nil+ }+ return false, fmt.Errorf("load pipeline %q for wiki alteration: %w", pipelineID, err)+ }+ return s.valueHasWikiTemplate(ctx, tenantID, canvas.DSL)+}++// valueHasWikiTemplate recursively inspects persisted parser/pipeline JSON.+// Pipeline DSLs and document parser configs use different nesting layouts, so+// looking only at a single top-level Compiler key misses pipeline documents.+func (s *DatasetArtifactService) valueHasWikiTemplate(ctx context.Context, tenantID string, value interface{}) (bool, error) {+ if items, ok := value.([]interface{}); ok {+ for _, item := range items {+ if found, err := s.valueHasWikiTemplate(ctx, tenantID, item); err != nil || found {+ return found, err+ }+ }+ return false, nil+ }+ params, isMap := value.(map[string]interface{})+ if !isMap {+ if typed, ok := value.(entity.JSONMap); ok {+ params = map[string]interface{}(typed)+ isMap = true+ }+ }+ if !isMap {+ return false, nil+ }+ if id, ok := params["compilation_template_id"].(string); ok && id != "" {+ var template entity.CompilationTemplate+ if err := dao.DB.WithContext(ctx).+ Where("id = ? AND (tenant_id = ? OR tenant_id IS NULL OR tenant_id = '') AND status = ?", id, tenantID, string(entity.StatusValid)).+ First(&template).Error; err != nil {+ if err == gorm.ErrRecordNotFound {+ return false, nil+ }+ return false, fmt.Errorf("load wiki compilation template %q: %w", id, err)+ }+ if templateKind(template) == "wiki" {+ return true, nil+ }+ }+ if groupID, ok := params["compilation_template_group_id"].(string); ok && groupID != "" {+ if found, err := s.groupHasWikiTemplate(ctx, tenantID, groupID); err != nil || found {+ return found, err+ }+ }+ if rawGroupIDs, ok := params["compilation_template_group_ids"]; ok {+ groupIDs := []string{}+ switch values := rawGroupIDs.(type) {+ case string:+ groupIDs = append(groupIDs, values)+ case []interface{}:+ for _, value := range values {+ if groupID, ok := value.(string); ok {+ groupIDs = append(groupIDs, groupID)+ }+ }+ case []string:+ groupIDs = values+ }+ for _, groupID := range groupIDs {+ if strings.TrimSpace(groupID) != "" {+ if found, err := s.groupHasWikiTemplate(ctx, tenantID, groupID); err != nil || found {+ return found, err+ }+ }+ }+ }+ for _, child := range params {+ if found, err := s.valueHasWikiTemplate(ctx, tenantID, child); err != nil || found {+ return found, err+ }+ }+ return false, nil+}++func (s *DatasetArtifactService) groupHasWikiTemplate(ctx context.Context, tenantID, groupID string) (bool, error) {+ var group entity.CompilationTemplateGroup+ if err := dao.DB.WithContext(ctx).+ Where("id = ? AND (tenant_id = ? OR tenant_id = '') AND status = ?", groupID, tenantID, string(entity.StatusValid)).+ First(&group).Error; err != nil {+ if err == gorm.ErrRecordNotFound {+ return false, nil+ }+ return false, fmt.Errorf("load compilation template group %q: %w", groupID, err)+ }+ var templates []entity.CompilationTemplate+ if err := dao.DB.WithContext(ctx).Where("group_id = ? AND status = ?", groupID, string(entity.StatusValid)).Find(&templates).Error; err != nil {+ return false, fmt.Errorf("load compilation template group %q: %w", groupID, err)+ }+ for _, template := range templates {+ if templateKind(template) == "wiki" {+ return true, nil+ }+ }+ return false, nil+}++func templateKind(template entity.CompilationTemplate) string {+ if kind, ok := template.Config["kind"].(string); ok && strings.TrimSpace(kind) != "" {+ return strings.ToLower(strings.TrimSpace(kind))+ }+ return strings.ToLower(strings.TrimSpace(template.Kind))+}++func setDifference(left, right map[string]struct{}) []string {+ ids := make([]string, 0)+ for id := range left {+ if _, ok := right[id]; !ok {+ ids = append(ids, id)+ }+ }+ sort.Strings(ids)+ return ids+}++func sortedSetKeys(set map[string]struct{}) []string {+ ids := make([]string, 0, len(set))+ for id := range set {+ ids = append(ids, id)+ }+ sort.Strings(ids)+ return ids+}+// ClearWiki deletes all wiki artifacts for a dataset.func (s *DatasetArtifactService) ClearWiki(ctx context.Context, tenantID, datasetID string) (map[string]int, error) {docEngine := engine.Get()@@ -792,6 +968,10 @@ func toStringSlice(v interface{}) []string {if t == "" {return []string{}}+ var decoded interface{}+ if json.Unmarshal([]byte(t), &decoded) == nil {+ return toStringSlice(decoded)+ }return []string{t}case []interface{}:out := make([]string, 0, len(t))diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.goindex 23edc266f5..17bdb9097a 100644--- a/internal/service/document/document_crud.go+++ b/internal/service/document/document_crud.go@@ -334,7 +334,18 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri}common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: failed to delete tasks for %s: %v", docID, delErr))}- return s.deleteDocRecordWithCounters(ctx, doc, kb.ID)+ if err := s.deleteDocRecordWithCounters(ctx, doc, kb.ID); err != nil {+ return err+ }+ // File replacement/deletion uses this path instead of deleteDocumentFull.+ // Publish the same deletion event so the dataset-level Wiki consumer removes+ // the deleted document's contribution in both paths.+ pubCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)+ defer cancel()+ if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID); err != nil {+ common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: publish doc_deleted for %s failed: %v", docID, err))+ }+ return nil}// InsertDocument creates a document row and increments the owning KB's doc_numdiff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.goindex d846994cfd..a5f19008f5 100644--- a/internal/service/document/document_dataset_update.go+++ b/internal/service/document/document_dataset_update.go@@ -80,13 +80,7 @@ func (s *DocumentService) BatchUpdateDocumentStatus(ctx context.Context, userID,hasError = truecontinue}- err = s.docEngine.UpdateChunks(- ctx,- map[string]interface{}{"doc_id": docID},- map[string]interface{}{"available_int": statusInt},- fmt.Sprintf("ragflow_%s", kb.TenantID),- doc.KbID,- )+ err = s.updateSourceChunkAvailability(ctx, kb.TenantID, doc.KbID, docID, statusInt)if err != nil {_ = s.documentDAO.UpdateByID(ctx, dao.DB, docID, map[string]interface{}{"status": previousStatus})msg := err.Error()@@ -99,6 +93,7 @@ func (s *DocumentService) BatchUpdateDocumentStatus(ctx context.Context, userID,continue}}+ s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, docID)result[docID] = map[string]string{"status": status}}diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.goindex b385dfbf72..9ba0823d06 100644--- a/internal/service/document/document_parse.go+++ b/internal/service/document/document_parse.go@@ -111,7 +111,7 @@ func (s *DocumentService) clearDocumentParseResults(ctx context.Context, doc *enif !exists {return nil}- if _, err = s.docEngine.DeleteChunks(ctx, map[string]interface{}{"doc_id": doc.ID}, indexName, doc.KbID); err != nil {+ if err = s.deleteSourceChunks(ctx, tenantID, doc.KbID, doc.ID); err != nil {return err}return nil@@ -405,7 +405,7 @@ func (s *DocumentService) resetDocumentForReparse(ctx context.Context, doc *entiif s.docEngine != nil {indexName := fmt.Sprintf("ragflow_%s", tenantID)s.deleteChunkImages(ctx, doc, indexName)- if _, err = s.docEngine.DeleteChunks(ctx, map[string]interface{}{"doc_id": doc.ID}, indexName, doc.KbID); err != nil {+ if err = s.deleteSourceChunks(ctx, tenantID, doc.KbID, doc.ID); err != nil {return err}}@@ -517,16 +517,11 @@ func (s *DocumentService) updateDocumentStatusOnly(ctx context.Context, doc *entreturn errors.New("database error (Document update)")}- if s.docEngine == nil {- return nil+ if s.docEngine != nil {+ if err := s.updateSourceChunkAvailability(ctx, kb.TenantID, doc.KbID, doc.ID, status); err != nil {+ return err+ }}-- indexName := fmt.Sprintf("ragflow_%s", kb.TenantID)- return s.docEngine.UpdateChunks(- ctx,- map[string]interface{}{"doc_id": doc.ID},- map[string]interface{}{"available_int": status},- indexName,- doc.KbID,- )+ s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, doc.ID)+ return nil}diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex 67e1c53e14..bc161f2b9a 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -26,6 +26,7 @@ import ("net/http""net/http/httptest""path/filepath"+ "reflect""strings""testing""time"@@ -227,6 +228,13 @@ func (e *rerunDeleteDocEngine) ChunkStoreExists(context.Context, string, string)return true, nil}+func (e *rerunDeleteDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {+ return &types.SearchResult{Chunks: []map[string]interface{}{+ {"id": "source-1"},+ {"id": "wiki-1", "compile_kwd": "wiki_page"},+ }, Total: 2}, nil+}+func (e *rerunDeleteDocEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {e.deleteCalls++e.condition = condition@@ -235,6 +243,24 @@ func (e *rerunDeleteDocEngine) DeleteChunks(_ context.Context, condition map[strreturn 3, nil}+type sourceAvailabilityDocEngine struct {+ fakeChatDocEngine+ updateConditions []map[string]interface{}+}++func (e *sourceAvailabilityDocEngine) Search(context.Context, *types.SearchRequest) (*types.SearchResult, error) {+ return &types.SearchResult{Chunks: []map[string]interface{}{+ {"id": "source-1"},+ {"id": "wiki-1", "compile_kwd": "wiki_page"},+ {"id": "map-1", "compile_kwd": []interface{}{"wiki_map_active"}},+ }, Total: 3}, nil+}++func (e *sourceAvailabilityDocEngine) UpdateChunks(_ context.Context, condition map[string]interface{}, _ map[string]interface{}, _, _ string) error {+ e.updateConditions = append(e.updateConditions, condition)+ return nil+}+type metadataDocEngine struct {fakeChatDocEnginerecords map[string]map[string]interface{}@@ -1951,11 +1977,28 @@ func TestClearDocumentParseResultsClearsCountersTasksAndChunks(t *testing.T) {if engine.deleteCalls != 1 {t.Fatalf("deleteCalls = %d, want 1", engine.deleteCalls)}- if engine.indexName != "ragflow_tenant-1" || engine.datasetID != "kb-1" || engine.condition["doc_id"] != "doc-1" {+ if engine.indexName != "ragflow_tenant-1" || engine.datasetID != "kb-1" || !reflect.DeepEqual(engine.condition["id"], []string{"source-1"}) {t.Fatalf("unexpected delete call: index=%s dataset=%s condition=%v", engine.indexName, engine.datasetID, engine.condition)}}+func TestUpdateSourceChunkAvailabilityExcludesCompiledProducts(t *testing.T) {+ docEngine := &sourceAvailabilityDocEngine{}+ svc := testDocumentService(t)+ svc.docEngine = docEngine++ if err := svc.updateSourceChunkAvailability(t.Context(), "tenant-1", "kb-1", "doc-1", 1); err != nil {+ t.Fatalf("updateSourceChunkAvailability failed: %v", err)+ }+ if len(docEngine.updateConditions) != 1 {+ t.Fatalf("UpdateChunks calls = %d, want 1", len(docEngine.updateConditions))+ }+ ids, ok := docEngine.updateConditions[0]["id"].([]string)+ if !ok || len(ids) != 1 || ids[0] != "source-1" {+ t.Fatalf("updated ids = %#v, want only source-1", docEngine.updateConditions[0]["id"])+ }+}+func TestClearDocumentParseResultsIsIdempotentForStaleDocSnapshot(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)diff --git a/internal/service/document/wiki_refresh.go b/internal/service/document/wiki_refresh.gonew file mode 100644index 0000000000..1e27a034fa--- /dev/null+++ b/internal/service/document/wiki_refresh.go@@ -0,0 +1,137 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package document++import (+ "context"+ "fmt"+ "strings"+ "time"++ "ragflow/internal/common"+ enginetypes "ragflow/internal/engine/types"+ "ragflow/internal/ingestion/knowledge_compile"++ "go.uber.org/zap"+)++const sourceChunkAvailabilityBatchSize = 1000++// updateSourceChunkAvailability changes retrieval visibility for source chunks+// only. Document-level compiler products must remain hidden (available_int=0)+// until the dataset-level compiler has merged them.+func (s *DocumentService) updateSourceChunkAvailability(ctx context.Context, tenantID, datasetID, documentID string, available int) error {+ if s.docEngine == nil {+ return fmt.Errorf("document engine not initialized")+ }+ indexName := fmt.Sprintf("ragflow_%s", tenantID)+ ids, err := s.loadSourceChunkIDs(ctx, indexName, datasetID, documentID)+ if err != nil {+ return err+ }+ for start := 0; start < len(ids); start += sourceChunkAvailabilityBatchSize {+ end := min(start+sourceChunkAvailabilityBatchSize, len(ids))+ if err := s.docEngine.UpdateChunks(ctx,+ map[string]any{"id": ids[start:end]},+ map[string]any{"available_int": available},+ indexName,+ datasetID,+ ); err != nil {+ return err+ }+ }+ return nil+}++func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, datasetID, documentID string) ([]string, error) {+ ids := make([]string, 0)+ for offset := 0; ; offset += sourceChunkAvailabilityBatchSize {+ result, err := s.docEngine.Search(ctx, &enginetypes.SearchRequest{+ IndexNames: []string{indexName},+ KbIDs: []string{datasetID},+ Offset: offset,+ Limit: sourceChunkAvailabilityBatchSize,+ SelectFields: []string{"id", "compile_kwd"},+ Filter: map[string]any{"doc_id": []string{documentID}},+ })+ if err != nil {+ return nil, err+ }+ if result == nil || len(result.Chunks) == 0 {+ break+ }+ for _, row := range result.Chunks {+ if strings.TrimSpace(documentStoreString(row["compile_kwd"])) != "" {+ continue+ }+ if id := strings.TrimSpace(documentStoreString(row["id"])); id != "" {+ ids = append(ids, id)+ }+ }+ if int64(offset+len(result.Chunks)) >= result.Total {+ break+ }+ }+ return ids, nil+}++func (s *DocumentService) deleteSourceChunks(ctx context.Context, tenantID, datasetID, documentID string) error {+ if s.docEngine == nil {+ return nil+ }+ indexName := fmt.Sprintf("ragflow_%s", tenantID)+ ids, err := s.loadSourceChunkIDs(ctx, indexName, datasetID, documentID)+ if err != nil || len(ids) == 0 {+ return err+ }+ for start := 0; start < len(ids); start += sourceChunkAvailabilityBatchSize {+ end := min(start+sourceChunkAvailabilityBatchSize, len(ids))+ if _, err := s.docEngine.DeleteChunks(ctx, map[string]any{+ "id": ids[start:end],+ "kb_id": datasetID,+ }, indexName, datasetID); err != nil {+ return err+ }+ }+ return nil+}++func (s *DocumentService) markDocumentWikiDirty(ctx context.Context, tenantID, datasetID, documentID string) {+ markCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)+ defer cancel()+ if err := knowledge_compile.MarkWikiDocumentDirty(markCtx, tenantID, datasetID, documentID, nil); err != nil {+ common.Warn("document mutation: failed to schedule Wiki refresh",+ zap.String("document_id", documentID), zap.Error(err))+ }+}++func documentStoreString(value any) string {+ switch typed := value.(type) {+ case string:+ return typed+ case []string:+ if len(typed) == 1 {+ return typed[0]+ }+ case []any:+ if len(typed) == 1 {+ if text, ok := typed[0].(string); ok {+ return text+ }+ }+ }+ return ""+}diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.pyindex 3a71d98fbc..81c0b6d087 100644--- a/rag/advanced_rag/knowlege_compile/wiki.py+++ b/rag/advanced_rag/knowlege_compile/wiki.py@@ -844,7 +844,7 @@ async def _wiki_scan_current_chunk_state(while True:res = await thread_pool_exec(settings.docStoreConn.search,- ["id", "doc_id", "content_with_weight"],+ ["id", "doc_id", "content_with_weight", "compile_kwd"],[],{"doc_id": [doc_id],@@ -858,8 +858,18 @@ async def _wiki_scan_current_chunk_state(index,[kb_id],)- rows = settings.docStoreConn.get_fields(res, ["id", "doc_id", "content_with_weight"]) or {}+ rows = settings.docStoreConn.get_fields(res, ["id", "doc_id", "content_with_weight", "compile_kwd"]) or {}for row_id, row in rows.items():+ # The source-chunk query also has a backend-level+ # ``must_not exists compile_kwd`` guard. Keep this explicit+ # check because some doc-store implementations do not apply+ # that nested filter consistently. Derived Wiki/structure+ # rows must never become MAP inputs or chunk deltas.+ compile_kwd = row.get("compile_kwd")+ if isinstance(compile_kwd, (list, tuple, set)):+ compile_kwd = next((str(value).strip() for value in compile_kwd if value), "")+ if str(compile_kwd or "").strip():+ continuechunk_id = str(row.get("id") or row_id or "")if chunk_id:state[chunk_id] = {@@ -912,6 +922,39 @@ async def _wiki_load_active_map_state(if len(rows) < 1000:breakoffset += 1000+ # Active snapshots are normally source-only because they are committed+ # from _wiki_scan_current_chunk_state. Older snapshots, however, may have+ # been created before derived rows were excluded. If a historical state ID+ # still exists in the index and carries compile_kwd, it is a compiled+ # product rather than an original chunk and must not be counted as a+ # deleted source chunk. Missing rows are intentionally retained: they are+ # the legitimate deleted/removed source chunks we need to report.+ chunk_ids = list(state)+ for start in range(0, len(chunk_ids), 500):+ batch_ids = chunk_ids[start : start + 500]+ try:+ res = await thread_pool_exec(+ settings.docStoreConn.search,+ ["id", "compile_kwd"],+ [],+ {"id": batch_ids},+ [],+ OrderByExpr(),+ 0,+ len(batch_ids),+ index,+ [kb_id],+ )+ rows = settings.docStoreConn.get_fields(res, ["id", "compile_kwd"]) or {}+ except Exception:+ logging.exception("wiki_map: failed to validate historical source snapshot")+ continue+ for row_id, row in rows.items():+ compile_kwd = row.get("compile_kwd")+ if isinstance(compile_kwd, (list, tuple, set)):+ compile_kwd = next((str(value).strip() for value in compile_kwd if value), "")+ if str(compile_kwd or "").strip():+ state.pop(str(row.get("id") or row_id), None)return statediff --git a/web/src/hooks/use-dataset-generate.ts b/web/src/hooks/use-dataset-generate.tsindex cfb168c9f6..86d3165a6d 100644--- a/web/src/hooks/use-dataset-generate.ts+++ b/web/src/hooks/use-dataset-generate.ts@@ -72,6 +72,7 @@ export interface ITraceInfo {// are the MySQL scheduling-entry counts, and error is the batch diagnostic// (NOT a peer state). Only populated by the Go/hybrid branch of useTraceQuery.compilationState?: string;+ currentPhase?: string;inflight?: number;backlog?: number;compilationError?: string;@@ -126,9 +127,10 @@ const useTraceQuery = (const state: string = st.state ?? 'idle';const error: string = st.error ?? '';return {- progress: state === 'completed' ? 1 : state === 'idle' ? 0 : 0,- progress_msg: error || state,+ progress: state === 'completed' ? 1 : (st.progress ?? 0),+ progress_msg: st.progress_msg || error || state,compilationState: state,+ currentPhase: st.current_phase ?? '',inflight: st.inflight ?? 0,backlog: st.backlog ?? 0,compilationError: error,@@ -283,11 +285,9 @@ export function useGenerateStatus(data?: ITraceInfo) {const percent = useMemo(() => {if (isGoDatasetBackend()) {- // No stable terminal state and no DocTotal/DocProcessed, so there is no- // meaningful percentage. Failures render as a full error marker; active- // runs show the inflight/backlog counts instead of a percent (see the- // UpdateRunProgress / EmptyState components).- return status === GenerateStatus.Failed ? 100 : 0;+ return status === GenerateStatus.Failed+ ? 100+ : (data?.progress ?? 0) * 100;}if (status === GenerateStatus.Failed) {return 100;diff --git a/web/src/pages/dataset/compilation/update-button.tsx b/web/src/pages/dataset/compilation/update-button.tsxindex 4ab9ed6f9c..b064a6c241 100644--- a/web/src/pages/dataset/compilation/update-button.tsx+++ b/web/src/pages/dataset/compilation/update-button.tsx@@ -11,7 +11,6 @@ import {} from '@/components/ui/tooltip';import { GenerateStatus, GenerateType } from '@/constants/knowledge';import { ITraceInfo, useGenerateStatus } from '@/hooks/use-dataset-generate';-import { isGoDatasetBackend } from '@/utils/api-proxy-scheme';import { UpdateRunProgress } from './update-run-progress';@@ -43,13 +42,6 @@ export function CompilationUpdateButton({const isRunning = status === GenerateStatus.Running;const isGenerating = isRunning || status === GenerateStatus.Failed;- // Go/hybrid: compilation is auto-driven by the scheduler with no manual- // re-merge, so hide the update control rather than offer a trigger that- // cannot work (plan v4.1 §4.2).- if (isGoDatasetBackend()) {- return null;- }-// A failed trace persists (progress stays < 0) until the next run, so it// must not keep the button visible on its own — only real changes or a// live run should.diff --git a/web/src/pages/dataset/compilation/update-run-progress.tsx b/web/src/pages/dataset/compilation/update-run-progress.tsxindex f778d1b57d..c5b2dd18ff 100644--- a/web/src/pages/dataset/compilation/update-run-progress.tsx+++ b/web/src/pages/dataset/compilation/update-run-progress.tsx@@ -46,7 +46,8 @@ export function UpdateRunProgress({<span className="size-2 rounded-full bg-accent-primary" />{data?.compilationError? data.compilationError- : t('knowledgeCompilation.compiling', {+ : data?.currentPhase ||+ t('knowledgeCompilation.compiling', {defaultValue: 'Compiling…',})}{!data?.compilationError && (