RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat: improve Go wiki compilation (#18479)
buua43631812e1de567
diff --git a/conf/infinity_mapping.json b/conf/infinity_mapping.jsonindex 2ab32c2bc1..95de7aeac4 100644--- a/conf/infinity_mapping.json+++ b/conf/infinity_mapping.json@@ -64,6 +64,8 @@"slug_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"title_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"topic_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},+ "plan_group_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},+ "generation_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"page_type_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},"entity_names_kwd": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},"entity_names": {"type": "json", "default": "[]", "enabled": false},diff --git a/internal/ingestion/component/knowledge_compiler/common/deps.go b/internal/ingestion/component/knowledge_compiler/common/deps.goindex 43caeff9d9..96111a0b41 100644--- a/internal/ingestion/component/knowledge_compiler/common/deps.go+++ b/internal/ingestion/component/knowledge_compiler/common/deps.go@@ -71,17 +71,26 @@ type HistoricalKNN interface {// WikiPageCandidate is one existing wiki page returned from the backing store.type WikiPageCandidate struct {- ID string- Slug string- Title string- PageType string- Topic string- Summary string- ContentMD string- ContentMDRaw string- EntityNames []string+ ID string+ Slug string+ Title string+ PageType string+ Topic string+ PlanGroup string+ Summary string+ ContentMD string+ ContentMDRaw string+ EntityNames []string+ // RoutedEntityNames is populated only when the topic route LLM explicitly+ // returns a new membership set. An empty value means preserve the existing+ // membership for backward-compatible direct-match routing.+ RoutedEntityNames []string+ // RoutedTopic is the topic selected by the topic route LLM. Empty means+ // preserve the existing topic value.+ RoutedTopic stringRelatedKBPages []stringOutlinks []string+ SourceChunkIDs []stringScore float64}@@ -93,6 +102,35 @@ type WikiPageStore interface {GetPageBySlug(ctx context.Context, tenantID, datasetID, slug string) (*WikiPageCandidate, error)}+// WikiPageCooccurrenceStore is an optional recall signal for topic routing.+// It returns pages whose source evidence contains one of the supplied chunks.+type WikiPageCooccurrenceStore interface {+ FindPagesBySourceChunks(ctx context.Context, tenantID, datasetID string, chunkIDs []string, k int) ([]WikiPageCandidate, error)+}++// WikiMapVersion identifies one immutable MAP extraction for a source chunk+// version. Key is a deterministic digest over the remaining identity fields;+// Payload is the variant-owned JSON representation of the extraction.+type WikiMapVersion struct {+ Key string+ TenantID string+ DatasetID string+ DocumentID string+ ChunkID string+ ContentHash string+ TemplateFingerprint string+ LLMFingerprint string+ Payload []byte+}++// WikiMapVersionStore persists immutable, reusable Wiki MAP results. Removing+// or disabling a source chunk does not delete its historical versions; the+// dataset semantic compiler decides which versions are active.+type WikiMapVersionStore interface {+ GetWikiMapVersions(ctx context.Context, tenantID, datasetID string, keys []string) (map[string][]byte, error)+ PutWikiMapVersions(ctx context.Context, versions []WikiMapVersion) error+}+// RedisClient is injected only for the datasetnav variant (M8).type RedisClient interface {// Minimal lock surface; full API added in M8.@@ -102,14 +140,15 @@ type RedisClient interface {// interfaces so tests inject stubs; production wiring lives in// internal/ingestion/task (see PORT_PLAN.md §4 dependency injection seam).type Deps struct {- Chat ChatInvoker- Embed Embedder- Tokenizer Tokenizer- HistoricalKNN HistoricalKNN // optional (wiki)- WikiPages WikiPageStore // optional (wiki)- Redis RedisClient // optional (datasetnav)- TenantID string- DatasetID string+ Chat ChatInvoker+ Embed Embedder+ Tokenizer Tokenizer+ HistoricalKNN HistoricalKNN // optional (wiki)+ WikiPages WikiPageStore // optional (wiki)+ WikiMapVersions WikiMapVersionStore // optional (wiki version cache)+ Redis RedisClient // optional (datasetnav)+ TenantID string+ DatasetID string// ModelContextLen is the chat model's context window in tokens// (content_length). The prompt-budget helpers (wikiMapMaxTokens,// deriveWikiPlanBudget, buildClusterContent) use it to size the input/outputdiff --git a/internal/ingestion/component/knowledge_compiler/wiki/page.go b/internal/ingestion/component/knowledge_compiler/wiki/page.goindex 74460cb83f..03b9e56fdf 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/page.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/page.go@@ -1,7 +1,6 @@package wikiimport (- "context""fmt""hash/fnv""strings"@@ -27,58 +26,7 @@ type wikiPageResult struct {Outlinks []stringSourceChunkIDs []stringSourceDocIDs []string-}--// mapFromChunks runs the MAP stage: extract raw facts/notes from the source.-func mapFromChunks(ctx context.Context, deps common.Deps, llmID, sections string) (string, error) {- resp, err := deps.Chat.Chat(ctx, common.ChatRequest{- LLMID: llmID,- SystemPrompt: wikiMapSystem,- UserPrompt: sections,- })- if err != nil {- return "", err- }- return resp.Content, nil-}--// reduceFromExtracts consolidates the MAP output into topic clusters.-func reduceFromExtracts(ctx context.Context, deps common.Deps, llmID, raw string) (string, error) {- resp, err := deps.Chat.Chat(ctx, common.ChatRequest{- LLMID: llmID,- SystemPrompt: wikiReduceSystem,- UserPrompt: raw,- })- if err != nil {- return "", err- }- return resp.Content, nil-}--// planFromReduction arranges topics into a page outline.-func planFromReduction(ctx context.Context, deps common.Deps, llmID, reduced string) (string, error) {- resp, err := deps.Chat.Chat(ctx, common.ChatRequest{- LLMID: llmID,- SystemPrompt: wikiPlanSystem,- UserPrompt: reduced,- })- if err != nil {- return "", err- }- return resp.Content, nil-}--// refineFromPlan writes the final wiki page from the outline.-func refineFromPlan(ctx context.Context, deps common.Deps, llmID, plan string) (string, error) {- resp, err := deps.Chat.Chat(ctx, common.ChatRequest{- LLMID: llmID,- SystemPrompt: wikiRefineSystem,- UserPrompt: plan,- })- if err != nil {- return "", err- }- return resp.Content, nil+ PlanGroup string}// Section is one heading block parsed out of a wiki page's Markdown body. The@@ -238,6 +186,7 @@ func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []com"source_chunk_ids": sourceChunkIDs,"source_doc_ids": sourceDocIDs,"content_md_raw": page.ContentRaw,+ "plan_group": strings.TrimSpace(page.PlanGroup),},})for i, sec := range ParseOutline(content) {@@ -266,6 +215,7 @@ func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []com"section_index": i,"source_chunk_ids": sourceChunkIDs,"source_doc_ids": sourceDocIDs,+ "plan_group": strings.TrimSpace(page.PlanGroup),},})}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go b/internal/ingestion/component/knowledge_compiler/wiki/prompt.goindex 25ddba76c7..95ef9d80e4 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/prompt.go@@ -9,12 +9,8 @@ import (const wikiMapSystem = `You are a knowledge extraction engine. Extract structured knowledge from the provided document sections. Return ONLY valid JSON matching the schema exactly. Never include text outside the JSON object. Keep the source language of the document.`-const wikiReduceSystem = `You are a knowledge synthesis engine. Normalize and deduplicate the structured extracts while preserving source_chunk_ids provenance. Claims, relations, and topics are pass-through facts; do not invent new ones. Return ONLY valid JSON matching the same schema.`-const wikiPlanSystem = `You are a knowledge compilation planner. Given structured knowledge, produce a wiki page plan. Return ONLY valid JSON.`-const wikiRefineSystem = `You are a technical writer. Write a complete wiki page from the plan, evidence checklist, and source text. Preserve factual density, keep the source language, and return only markdown.`-const wikiReduceEntityDisambiguateSystem = `You are a knowledge canonicalization engine. Decide whether two named entities refer to the same real-world concept. Return ONLY valid JSON.`const wikiReduceEntityDisambiguateUserTemplate = `## Entity A@@ -187,6 +183,8 @@ Return JSON:{"action": "UPDATE | CREATE","slug": "string",+ "topic": "string",+ "entity_names": ["string"],"reason": "string"}@@ -194,6 +192,11 @@ Rules:- Choose UPDATE only when the planned page and candidate refer to the same underlying entity, concept, or topic.- Prefer CREATE when the overlap is weak, ambiguous, or just generally related.- If action is UPDATE, slug must be exactly one candidate slug.+- If action is UPDATE, entity_names must contain the complete membership set+ for the target page. Remove members that no longer belong and include newly+ routed members. If membership is unchanged, repeat the candidate members.+- topic must be the final topic label for the page. If action is CREATE, use a+ concise topic label grounded in the planned page and candidates.- Return only JSON.`const wikiRefineWriterExample = `Each page must be a proper encyclopedic article, not a flat bullet list:diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex a004e49223..de9db1975c 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -1,6 +1,7 @@// Package wiki implements the "wiki" variant of KnowledgeCompiler: a// document artifact pipeline MAP -> REDUCE -> PLAN -> REFINE that produces a-// wiki-style page (and supporting section products) with in-memory state only.+// wiki-style page (and supporting section products). MAP results may be reused+// from the immutable per-chunk version store; later stages remain in memory.// The stage semantics are aligned with the Python wiki.py design, but the Go// port keeps all intermediate artifacts in memory instead of persisting them to// ES between stages.@@ -103,6 +104,7 @@ type wikiPipeline struct {// planCapacityExcluded counts the planned pages dropped to fit the global// hard cap; testable and reported for observability.planCapacityExcluded int+ neighborCache map[string]*common.WikiPageCandidate}type wikiExtract struct {@@ -111,6 +113,7 @@ type wikiExtract struct {Claims []wikiClaim `json:"claims"`Relations []wikiRelation `json:"relations"`Topics []string `json:"topics"`+ Mode string `json:"mode,omitempty"`}type wikiEntity struct {@@ -171,7 +174,8 @@ type wikiPlanPage struct {// MentionCount is an internal (non-serialized) signal used for the// deterministic page selection when the merged plan exceeds the global hard// cap. It is computed from the reduced extract, not read from JSON.- MentionCount int `json:"-"`+ MentionCount int `json:"-"`+ PlanGroup string `json:"plan_group,omitempty"`}// Reconciliation thresholds. These are a deliberate Go-specific refinement of@@ -216,14 +220,15 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commozap.Bool("chat_ready", deps.Chat != nil),zap.Bool("embed_ready", deps.Embed != nil))p := &wikiPipeline{- ctx: ctx,- deps: deps,- param: param,- inputs: inputs,- docID: docID,- tenantID: deps.TenantID,- datasetID: deps.DatasetID,- llmID: llmID,+ ctx: ctx,+ deps: deps,+ param: param,+ inputs: inputs,+ docID: docID,+ tenantID: deps.TenantID,+ datasetID: deps.DatasetID,+ llmID: llmID,+ neighborCache: make(map[string]*common.WikiPageCandidate),}if err := p.run(); err != nil {appcommon.Error("wiki: Run pipeline failed", err,@@ -538,6 +543,13 @@ func pageResultsDebug(pages []wikiPageResult) []string {}func (p *wikiPipeline) runMap() error {+ if p.deps.WikiMapVersions != nil {+ err := p.runVersionedMap()+ for i := range p.mapExtracts {+ p.mapExtracts[i].Mode = p.wikiMode()+ }+ return err+ }// Keep each batch small enough that the LLM's entity/relation JSON output// for the batch stays well under the model's output-token limit. 2048 input// tokens per batch (was a hard-coded 4096) leaves generous headroom for the@@ -549,9 +561,19 @@ func (p *wikiPipeline) runMap() error {return err}p.mapExtracts = append(p.mapExtracts, extracts...)+ for i := range p.mapExtracts {+ p.mapExtracts[i].Mode = p.wikiMode()+ }return nil}+func (p *wikiPipeline) wikiMode() string {+ if p.param.PlanEnabled() {+ return "topic"+ }+ return "entity"+}+func runMapBatches(ctx context.Context,batches [][]common.Chunk,@@ -602,10 +624,19 @@ func (p *wikiPipeline) mapBatch(batch []common.Chunk) (wikiExtract, error) {if err != nil {return wikiExtract{}, err}- return parseWikiExtract(raw), nil+ extract := parseWikiExtract(raw)+ extract.Mode = p.wikiMode()+ return extract, nil}func (p *wikiPipeline) runPlan() (wikiPlan, error) {+ if p.wikiMode() == "topic" && p.deps.Embed != nil {+ return p.runTopicPlan()+ }+ return p.runLegacyPlan()+}++func (p *wikiPipeline) runLegacyPlan() (wikiPlan, error) {batches := packWikiPlanBatches(p.reduced, wikiPlanTokenBudget)if len(batches) == 0 {batches = []wikiExtract{p.reduced}@@ -837,6 +868,7 @@ func (p *wikiPipeline) runRefinePage(Outlinks: outlinks,SourceChunkIDs: sourceChunkIDs,SourceDocIDs: sourceDocIDs,+ PlanGroup: planItem.PlanGroup,}, nil}@@ -964,9 +996,19 @@ func (p *wikiPipeline) reconcilePlan(plan wikiPlan) (wikiPlan, error) {page.Slug = firstNonEmpty(existing.Slug, page.Slug)page.Title = firstNonEmpty(existing.Title, page.Title)page.PageType = firstNonEmpty(existing.PageType, page.PageType)- page.Topic = firstNonEmpty(existing.Topic, page.Topic)+ if existing.RoutedTopic != "" {+ page.Topic = existing.RoutedTopic+ } else {+ page.Topic = firstNonEmpty(existing.Topic, page.Topic)+ }page.RelatedKB = mergeStrings(page.RelatedKB, existing.RelatedKBPages)- page.EntityNames = mergeStrings(page.EntityNames, existing.EntityNames)+ if len(existing.RoutedEntityNames) > 0 {+ // An explicit route membership set is authoritative: this is a+ // migration, so do not union members that the LLM removed.+ page.EntityNames = uniqueStrings(existing.RoutedEntityNames)+ } else {+ page.EntityNames = mergeStrings(page.EntityNames, existing.EntityNames)+ }plan.Pages[idx] = page}plan.Pages = normalizeWikiPlanPages(plan.Pages, p.reduced)@@ -989,6 +1031,11 @@ func (p *wikiPipeline) reconcilePlanPage(page wikiPlanPage, queryVec []float32)if err != nil || len(cands) == 0 {return nil, err}+ // Topic incremental routing uses more than page-vector similarity: pages+ // linked from the KNN candidates are added as semantic neighbors, so a+ // related page whose title is not embedding-similar can still be considered+ // by the LLM route decision.+ cands = p.expandTopicNeighborCandidates(page, cands)cands = rerankWikiPlanCandidates(page, cands)targetNames := normalizedStringSet(page.EntityNames)pageTitle := normKey(page.Title)@@ -1016,6 +1063,110 @@ func (p *wikiPipeline) reconcilePlanPage(page wikiPlanPage, queryVec []float32)return nil, nil}+func (p *wikiPipeline) expandTopicNeighborCandidates(page wikiPlanPage, candidates []common.WikiPageCandidate) []common.WikiPageCandidate {+ if p.deps.WikiPages == nil {+ return candidates+ }+ if p.neighborCache == nil {+ p.neighborCache = make(map[string]*common.WikiPageCandidate)+ }+ out := append([]common.WikiPageCandidate(nil), candidates...)+ seen := make(map[string]struct{}, len(out))+ for _, candidate := range out {+ seen[candidate.Slug] = struct{}{}+ }+ for _, candidate := range candidates {+ for _, slug := range candidate.RelatedKBPages {+ if len(out) >= len(candidates)+wikiTopicNeighborMax {+ break+ }+ slug = strings.TrimSpace(slug)+ if slug == "" {+ continue+ }+ if _, exists := seen[slug]; exists {+ continue+ }+ var neighbor *common.WikiPageCandidate+ if cached, ok := p.neighborCache[slug]; ok {+ neighbor = cached+ } else {+ loaded, err := p.deps.WikiPages.GetPageBySlug(p.ctx, p.tenantID, p.datasetID, slug)+ if err != nil {+ p.neighborCache[slug] = nil+ continue+ }+ neighbor = loaded+ p.neighborCache[slug] = loaded+ }+ if neighbor == nil {+ continue+ }+ seen[neighbor.Slug] = struct{}{}+ neighbor.Score = candidate.Score * 0.95+ out = append(out, *neighbor)+ }+ }+ if store, ok := p.deps.WikiPages.(common.WikiPageCooccurrenceStore); ok {+ if chunks := p.sourceChunksForPlanPage(page); len(chunks) > 0 {+ cooccurring, err := store.FindPagesBySourceChunks(p.ctx, p.tenantID, p.datasetID, chunks, wikiPlanSearchTopK)+ if err == nil {+ for _, candidate := range cooccurring {+ if candidate.Slug == "" {+ continue+ }+ if _, exists := seen[candidate.Slug]; exists {+ continue+ }+ candidate.Score = 0.68+ seen[candidate.Slug] = struct{}{}+ out = append(out, candidate)+ }+ }+ }+ }+ return out+}++func (p *wikiPipeline) sourceChunksForPlanPage(page wikiPlanPage) []string {+ names := normalizedStringSet(page.EntityNames)+ for _, name := range []string{page.Title, page.Topic} {+ if key := normKey(name); key != "" {+ names[key] = struct{}{}+ }+ }+ chunks := make([]string, 0)+ for _, entity := range p.reduced.Entities {+ if _, ok := names[normKey(entity.Name)]; ok {+ chunks = append(chunks, entity.SourceChunkIDs...)+ }+ }+ for _, concept := range p.reduced.Concepts {+ if _, ok := names[normKey(concept.Term)]; ok {+ chunks = append(chunks, concept.SourceChunkIDs...)+ }+ }+ for _, claim := range p.reduced.Claims {+ if _, ok := names[normKey(claim.Subject)]; ok {+ chunks = append(chunks, claim.SourceChunkIDs...)+ }+ }+ for _, relation := range p.reduced.Relations {+ if _, fromOK := names[normKey(relation.From)]; fromOK {+ chunks = append(chunks, relation.SourceChunkIDs...)+ continue+ }+ if _, toOK := names[normKey(relation.To)]; toOK {+ chunks = append(chunks, relation.SourceChunkIDs...)+ }+ }+ chunks = uniqueStrings(chunks)+ if len(chunks) > wikiTopicSourceChunkMax {+ chunks = chunks[:wikiTopicSourceChunkMax]+ }+ return chunks+}+func rerankWikiPlanCandidates(page wikiPlanPage, candidates []common.WikiPageCandidate) []common.WikiPageCandidate {if len(candidates) <= 1 {return candidates@@ -1083,6 +1234,12 @@ func (p *wikiPipeline) resolveMaybePlanPage(page wikiPlanPage, candidates []comm}for i := range candidates {if candidates[i].Slug == slug {+ if topic := strings.TrimSpace(firstString(raw["topic"])); topic != "" {+ candidates[i].RoutedTopic = topic+ }+ if names := parseWikiStrings(raw["entity_names"]); len(names) > 0 {+ candidates[i].RoutedEntityNames = names+ }return &candidates[i], nil}}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.gonew file mode 100644index 0000000000..47deab1bce--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go@@ -0,0 +1,263 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package wiki++import (+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "sort"+ "strings"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++type wikiMapChunkVersion struct {+ index int+ chunk common.Chunk+ key string+ contentHash string+}++// runVersionedMap reuses immutable per-chunk MAP results and only submits cache+// misses to the LLM. Results are split back to one payload per source chunk so+// a later active snapshot can add or retract evidence at chunk granularity.+func (p *wikiPipeline) runVersionedMap() error {+ templateFingerprint, err := p.wikiMapTemplateFingerprint()+ if err != nil {+ return err+ }+ llmFingerprint := wikiMapHash(p.llmID)++ versions := make([]wikiMapChunkVersion, 0, len(p.inputs.Chunks))+ keys := make([]string, 0, len(p.inputs.Chunks))+ for i, chunk := range p.inputs.Chunks {+ if strings.TrimSpace(chunk.ID) == "" {+ continue+ }+ contentHash := wikiMapHash(firstNonEmpty(chunk.Text, chunk.Content))+ key := wikiMapHash(strings.Join([]string{+ p.tenantID,+ p.datasetID,+ p.docID,+ chunk.ID,+ contentHash,+ templateFingerprint,+ llmFingerprint,+ }, "\x00"))+ versions = append(versions, wikiMapChunkVersion{index: i, chunk: chunk, key: key, contentHash: contentHash})+ keys = append(keys, key)+ }++ cached, err := p.deps.WikiMapVersions.GetWikiMapVersions(p.ctx, p.tenantID, p.datasetID, keys)+ if err != nil {+ return fmt.Errorf("wiki: load MAP versions: %w", err)+ }++ extracts := make([]wikiExtract, len(p.inputs.Chunks))+ resolved := make([]bool, len(p.inputs.Chunks))+ misses := make([]common.Chunk, 0, len(versions))+ toSave := make([]common.WikiMapVersion, 0, len(versions))+ versionByChunkID := make(map[string]wikiMapChunkVersion, len(versions))+ for _, version := range versions {+ versionByChunkID[version.chunk.ID] = version+ payload, ok := cached[version.key]+ if !ok {+ misses = append(misses, version.chunk)+ continue+ }+ if err := json.Unmarshal(payload, &extracts[version.index]); err != nil {+ return fmt.Errorf("wiki: decode MAP version %s for chunk %s: %w", version.key, version.chunk.ID, err)+ }+ // Stamp the effective compiler mode on the in-memory extract. The mode is+ // part of the dataset FINALIZE contract and cache rows remain immutable.+ extracts[version.index].Mode = p.wikiMode()+ resolved[version.index] = true+ }++ batches := common.PackBatches(misses, wikiMapTokenBudget, p.deps.Tokenizer)+ batchExtracts, err := runMapBatches(p.ctx, batches, p.mapBatch)+ if err != nil {+ return err+ }+ for i, batch := range batches {+ if i >= len(batchExtracts) {+ return fmt.Errorf("wiki: MAP batch result missing at index %d", i)+ }+ perChunk := splitWikiExtractByChunk(batchExtracts[i], batch)+ for _, chunk := range batch {+ version, ok := versionByChunkID[chunk.ID]+ if !ok {+ continue+ }+ extract := perChunk[chunk.ID]+ payload, err := json.Marshal(extract)+ if err != nil {+ return fmt.Errorf("wiki: encode MAP version for chunk %s: %w", chunk.ID, err)+ }+ extracts[version.index] = extract+ resolved[version.index] = true+ toSave = append(toSave, common.WikiMapVersion{+ Key: version.key,+ TenantID: p.tenantID,+ DatasetID: p.datasetID,+ DocumentID: p.docID,+ ChunkID: chunk.ID,+ ContentHash: version.contentHash,+ TemplateFingerprint: templateFingerprint,+ LLMFingerprint: llmFingerprint,+ Payload: payload,+ })+ }+ }+ if err := p.deps.WikiMapVersions.PutWikiMapVersions(p.ctx, toSave); err != nil {+ return fmt.Errorf("wiki: save MAP versions: %w", err)+ }++ // Chunks without a stable id cannot form a reusable cache identity. Keep the+ // old batched behavior for them instead of inventing positional ids.+ var uncacheable []common.Chunk+ for i, chunk := range p.inputs.Chunks {+ if resolved[i] {+ continue+ }+ uncacheable = append(uncacheable, chunk)+ }+ uncacheableBatches := common.PackBatches(uncacheable, wikiMapTokenBudget, p.deps.Tokenizer)+ uncachedExtracts, err := runMapBatches(p.ctx, uncacheableBatches, p.mapBatch)+ if err != nil {+ return err+ }++ for _, extract := range extracts {+ if wikiExtractHasContent(extract) {+ p.mapExtracts = append(p.mapExtracts, extract)+ }+ }+ p.mapExtracts = append(p.mapExtracts, uncachedExtracts...)+ return nil+}++func wikiExtractHasContent(extract wikiExtract) bool {+ return len(extract.Entities) > 0 || len(extract.Concepts) > 0 || len(extract.Claims) > 0 || len(extract.Relations) > 0 || len(extract.Topics) > 0+}++func (p *wikiPipeline) wikiMapTemplateFingerprint() (string, error) {+ mapConfig := p.param.TemplateConfig+ if configured, ok := p.inputs.VariantSpecific["parser_config"].(map[string]any); ok {+ mapConfig = configured+ }+ payload, err := json.Marshal(struct {+ TemplateID string `json:"template_id"`+ TemplateConfig map[string]any `json:"template_config"`+ Language string `json:"language"`+ SystemPrompt string `json:"system_prompt"`+ UserTemplate string `json:"user_template"`+ }{+ TemplateID: p.param.TemplateID,+ TemplateConfig: mapConfig,+ Language: p.param.Language,+ SystemPrompt: wikiMapSystem,+ UserTemplate: wikiMapUserTemplate,+ })+ if err != nil {+ return "", fmt.Errorf("wiki: fingerprint MAP template: %w", err)+ }+ return wikiMapHash(string(payload)), nil+}++func wikiMapHash(value string) string {+ sum := sha256.Sum256([]byte(value))+ return hex.EncodeToString(sum[:])+}++func splitWikiExtractByChunk(extract wikiExtract, batch []common.Chunk) map[string]wikiExtract {+ out := make(map[string]wikiExtract, len(batch))+ known := make(map[string]struct{}, len(batch))+ for _, chunk := range batch {+ if chunk.ID != "" {+ known[chunk.ID] = struct{}{}+ out[chunk.ID] = wikiExtract{}+ }+ }++ for _, entity := range extract.Entities {+ for _, chunkID := range wikiMapItemChunkIDs(entity.SourceChunkIDs, known) {+ part := out[chunkID]+ entityCopy := entity+ entityCopy.SourceChunkIDs = []string{chunkID}+ part.Entities = append(part.Entities, entityCopy)+ out[chunkID] = part+ }+ }+ for _, concept := range extract.Concepts {+ for _, chunkID := range wikiMapItemChunkIDs(concept.SourceChunkIDs, known) {+ part := out[chunkID]+ conceptCopy := concept+ conceptCopy.SourceChunkIDs = []string{chunkID}+ part.Concepts = append(part.Concepts, conceptCopy)+ out[chunkID] = part+ }+ }+ for _, claim := range extract.Claims {+ for _, chunkID := range wikiMapItemChunkIDs(claim.SourceChunkIDs, known) {+ part := out[chunkID]+ claimCopy := claim+ claimCopy.SourceChunkIDs = []string{chunkID}+ part.Claims = append(part.Claims, claimCopy)+ out[chunkID] = part+ }+ }+ for _, relation := range extract.Relations {+ for _, chunkID := range wikiMapItemChunkIDs(relation.SourceChunkIDs, known) {+ part := out[chunkID]+ relationCopy := relation+ relationCopy.SourceChunkIDs = []string{chunkID}+ part.Relations = append(part.Relations, relationCopy)+ out[chunkID] = part+ }+ }+ for chunkID, part := range out {+ part.Topics = append(part.Topics, extract.Topics...)+ out[chunkID] = part+ }+ return out+}++func wikiMapItemChunkIDs(sourceChunkIDs []string, known map[string]struct{}) []string {+ if len(sourceChunkIDs) == 0 {+ out := make([]string, 0, len(known))+ for chunkID := range known {+ out = append(out, chunkID)+ }+ sort.Strings(out)+ return out+ }+ out := make([]string, 0, len(sourceChunkIDs))+ seen := make(map[string]struct{}, len(sourceChunkIDs))+ for _, chunkID := range sourceChunkIDs {+ if _, ok := known[chunkID]; !ok {+ continue+ }+ if _, duplicate := seen[chunkID]; duplicate {+ continue+ }+ seen[chunkID] = struct{}{}+ out = append(out, chunkID)+ }+ return out+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache_test.gonew file mode 100644index 0000000000..8a890e6eb9--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache_test.go@@ -0,0 +1,210 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package wiki++import (+ "context"+ "strings"+ "sync"+ "testing"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++type memoryWikiMapVersionStore struct {+ mu sync.Mutex+ payloads map[string][]byte+}++func (s *memoryWikiMapVersionStore) GetWikiMapVersions(_ context.Context, _, _ string, keys []string) (map[string][]byte, error) {+ s.mu.Lock()+ defer s.mu.Unlock()+ out := make(map[string][]byte, len(keys))+ for _, key := range keys {+ if payload, ok := s.payloads[key]; ok {+ out[key] = append([]byte(nil), payload...)+ }+ }+ return out, nil+}++func (s *memoryWikiMapVersionStore) PutWikiMapVersions(_ context.Context, versions []common.WikiMapVersion) error {+ s.mu.Lock()+ defer s.mu.Unlock()+ if s.payloads == nil {+ s.payloads = map[string][]byte{}+ }+ for _, version := range versions {+ if _, exists := s.payloads[version.Key]; !exists {+ s.payloads[version.Key] = append([]byte(nil), version.Payload...)+ }+ }+ return nil+}++type countingWikiMapChat struct {+ mu sync.Mutex+ calls int+}++type staticWikiEmbedder struct{}++func (staticWikiEmbedder) Encode(_ context.Context, texts []string) ([][]float32, error) {+ vectors := make([][]float32, len(texts))+ for i := range texts {+ vectors[i] = []float32{1, 0}+ }+ return vectors, nil+}++func (staticWikiEmbedder) Dimensions() int { return 2 }++func (c *countingWikiMapChat) Chat(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {+ c.mu.Lock()+ c.calls+++ c.mu.Unlock()+ if strings.Contains(req.SystemPrompt, "knowledge compilation writer") {+ return &common.ChatResponse{Content: "# Alpha\n\nAlpha exists.\n"}, nil+ }+ return &common.ChatResponse{Content: `{+ "entities": [{"name":"Alpha","type":"thing","source_chunk_ids":["c1"]}],+ "concepts": [],+ "claims": [{"statement":"Alpha exists","subject":"Alpha","confidence":"explicit","source_chunk_ids":["c1"]}],+ "relations": [],+ "topics": ["Alpha"]+}`}, nil+}++func (c *countingWikiMapChat) callCount() int {+ c.mu.Lock()+ defer c.mu.Unlock()+ return c.calls+}++func TestRunVersionedMapReusesHistoricalChunkVersion(t *testing.T) {+ store := &memoryWikiMapVersionStore{payloads: map[string][]byte{}}+ chat := &countingWikiMapChat{}+ newPipeline := func(text string) *wikiPipeline {+ return &wikiPipeline{+ ctx: t.Context(),+ deps: common.Deps{Chat: chat, WikiMapVersions: store},+ param: common.Param{TemplateID: "tpl-1", Language: "English", TemplateConfig: map[string]any{"mode": "entity"}},+ inputs: common.Inputs{Chunks: []common.Chunk{{ID: "c1", Text: text}}},+ docID: "doc-1",+ tenantID: "tenant-1",+ datasetID: "kb-1",+ llmID: "llm-1",+ }+ }++ first := newPipeline("version A")+ if err := first.runMap(); err != nil {+ t.Fatalf("first runMap() error = %v", err)+ }+ if got := chat.callCount(); got != 1 {+ t.Fatalf("first MAP calls = %d, want 1", got)+ }+ if len(first.mapExtracts) != 1 || len(first.mapExtracts[0].Entities) != 1 {+ t.Fatalf("first MAP extracts = %#v", first.mapExtracts)+ }++ second := newPipeline("version B")+ if err := second.runMap(); err != nil {+ t.Fatalf("second runMap() error = %v", err)+ }+ if got := chat.callCount(); got != 2 {+ t.Fatalf("changed-content MAP calls = %d, want 2", got)+ }++ third := newPipeline("version A")+ if err := third.runMap(); err != nil {+ t.Fatalf("third runMap() error = %v", err)+ }+ if got := chat.callCount(); got != 2 {+ t.Fatalf("historical-version MAP calls = %d, want 2", got)+ }+ if len(store.payloads) != 2 {+ t.Fatalf("stored MAP versions = %d, want 2", len(store.payloads))+ }+}++func TestRunWithVersionedMapStoreStillBuildsDocumentPages(t *testing.T) {+ store := &memoryWikiMapVersionStore{payloads: map[string][]byte{}}+ chat := &countingWikiMapChat{}+ deps := common.Deps{+ TenantID: "tenant-1", DatasetID: "kb-1",+ Chat: chat, Embed: staticWikiEmbedder{}, WikiMapVersions: store,+ }+ param := common.Param{}.Defaults()+ param.TemplateID = "tpl-1"+ param.LLMID = "llm-1"+ param.Language = "English"+ param.TemplateConfig = map[string]any{"mode": "entity"}+ inputs := common.Inputs{+ DocID: "doc-1", Chunks: []common.Chunk{{ID: "c1", Text: "Alpha exists."}},+ }+ pipeline := &wikiPipeline{+ ctx: t.Context(), deps: deps, param: param, inputs: inputs,+ docID: "doc-1", tenantID: "tenant-1", datasetID: "kb-1", llmID: "llm-1",+ }+ if err := pipeline.run(); err != nil {+ t.Fatalf("pipeline.run() error = %v", err)+ }+ if len(pipeline.pages) == 0 {+ t.Fatalf("pipeline produced no pages: map=%d entities=%d plan=%d", len(pipeline.mapExtracts), len(pipeline.reduced.Entities), len(pipeline.plan.Pages))+ }+ out, err := Run(t.Context(), deps, param, inputs)+ if err != nil {+ t.Fatalf("Run() error = %v", err)+ }+ if len(store.payloads) != 1 {+ t.Fatalf("stored MAP versions = %d, want 1", len(store.payloads))+ }+ if len(out.Products) == 0 {+ t.Fatal("Run() returned no document Wiki pages")+ }+ for _, product := range out.Products {+ if kind, _ := product.Meta["kind"].(string); kind != "page" && kind != "section" {+ t.Fatalf("Run() returned intermediate product kind %q", kind)+ }+ if len(product.Vector) == 0 {+ t.Fatalf("Wiki product %s has no embedding", product.ID)+ }+ }+}++func TestSplitWikiExtractByChunkDropsUnknownProvenance(t *testing.T) {+ parts := splitWikiExtractByChunk(wikiExtract{+ Entities: []wikiEntity{+ {Name: "Alpha", SourceChunkIDs: []string{"c1", "unknown"}},+ {Name: "Ghost", SourceChunkIDs: []string{"unknown"}},+ },+ Claims: []wikiClaim{{Statement: "shared", Subject: "Alpha"}},+ }, []common.Chunk{{ID: "c1"}, {ID: "c2"}})++ if got := len(parts["c1"].Entities); got != 1 {+ t.Fatalf("c1 entities = %d, want 1", got)+ }+ if got := len(parts["c2"].Entities); got != 0 {+ t.Fatalf("c2 entities = %d, want 0", got)+ }+ if got := len(parts["c1"].Claims); got != 1 {+ t.Fatalf("c1 claims = %d, want source-less claim fallback", got)+ }+ if got := len(parts["c2"].Claims); got != 1 {+ t.Fatalf("c2 claims = %d, want source-less claim fallback", got)+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.gonew file mode 100644index 0000000000..77f691b22f--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go@@ -0,0 +1,191 @@+package wiki++import (+ "fmt"+ "sort"+ "strings"++ appcommon "ragflow/internal/common"++ "go.uber.org/zap"+)++const (+ wikiTopicCandidateThreshold = 0.72+ wikiTopicCommunityMaxItems = 12+ wikiTopicNeighborMax = 32+ wikiTopicSourceChunkMax = 128+)++type topicCommunityItem struct {+ name string+ entity *wikiEntity+ concept *wikiConcept+ vector []float32+}++// buildTopicCandidateCommunities uses embedding as a candidate-windowing+// signal only. The final page membership is still decided by the LLM planner.+func (p *wikiPipeline) buildTopicCandidateCommunities() []wikiExtract {+ items := make([]topicCommunityItem, 0, len(p.reduced.Entities)+len(p.reduced.Concepts))+ for i := range p.reduced.Entities {+ entity := p.reduced.Entities[i]+ if name := strings.TrimSpace(entity.Name); name != "" {+ items = append(items, topicCommunityItem{name: name, entity: &entity})+ }+ }+ for i := range p.reduced.Concepts {+ concept := p.reduced.Concepts[i]+ if name := strings.TrimSpace(concept.Term); name != "" {+ items = append(items, topicCommunityItem{name: name, concept: &concept})+ }+ }+ sort.SliceStable(items, func(i, j int) bool { return normKey(items[i].name) < normKey(items[j].name) })+ if len(items) < 2 || p.deps.Embed == nil {+ return []wikiExtract{p.reduced}+ }+ texts := make([]string, len(items))+ for i := range items {+ texts[i] = items[i].name+ }+ vectors, err := p.deps.Embed.Encode(p.ctx, texts)+ if err != nil || len(vectors) != len(items) {+ return []wikiExtract{p.reduced}+ }+ for i := range items {+ items[i].vector = vectors[i]+ }+ communities := make([][]topicCommunityItem, 0)+ for _, item := range items {+ placed := -1+ best := -1.0+ for i, community := range communities {+ if len(community) >= wikiTopicCommunityMaxItems {+ continue+ }+ for _, member := range community {+ score := cosine32(item.vector, member.vector)+ if score >= wikiTopicCandidateThreshold && score > best {+ best = score+ placed = i+ }+ }+ }+ if placed < 0 {+ communities = append(communities, []topicCommunityItem{item})+ } else {+ communities[placed] = append(communities[placed], item)+ }+ }+ residual := wikiExtract{}+ assignedClaims := make([]bool, len(p.reduced.Claims))+ assignedRelations := make([]bool, len(p.reduced.Relations))+ assignedTopics := make([]bool, len(p.reduced.Topics))+ out := make([]wikiExtract, 0, len(communities))+ for i, community := range communities {+ nameSet := make(map[string]struct{}, len(community))+ extract := wikiExtract{}+ for _, item := range community {+ nameSet[normKey(item.name)] = struct{}{}+ if item.entity != nil {+ extract.Entities = append(extract.Entities, *item.entity)+ }+ if item.concept != nil {+ extract.Concepts = append(extract.Concepts, *item.concept)+ }+ }+ for claimIndex, claim := range p.reduced.Claims {+ if _, ok := nameSet[normKey(claim.Subject)]; ok {+ extract.Claims = append(extract.Claims, claim)+ assignedClaims[claimIndex] = true+ } else if i == len(communities)-1 && !assignedClaims[claimIndex] {+ residual.Claims = append(residual.Claims, claim)+ }+ }+ for relationIndex, relation := range p.reduced.Relations {+ _, fromOK := nameSet[normKey(relation.From)]+ _, toOK := nameSet[normKey(relation.To)]+ if fromOK || toOK {+ extract.Relations = append(extract.Relations, relation)+ assignedRelations[relationIndex] = true+ } else if i == len(communities)-1 && !assignedRelations[relationIndex] {+ residual.Relations = append(residual.Relations, relation)+ }+ }+ for topicIndex, topic := range p.reduced.Topics {+ if _, ok := nameSet[normKey(topic)]; ok {+ extract.Topics = append(extract.Topics, topic)+ assignedTopics[topicIndex] = true+ } else if i == len(communities)-1 && !assignedTopics[topicIndex] {+ residual.Topics = append(residual.Topics, topic)+ }+ }+ extract.Topics = uniqueStrings(extract.Topics)+ out = append(out, extract)+ }+ if len(residual.Claims) > 0 || len(residual.Relations) > 0 || len(residual.Topics) > 0 {+ residual.Topics = uniqueStrings(residual.Topics)+ out = append(out, residual)+ }+ return out+}++func (p *wikiPipeline) runTopicPlan() (wikiPlan, error) {+ communities := p.buildTopicCandidateCommunities()+ if len(communities) == 0 {+ return wikiPlan{}, nil+ }+ totalItems := 0+ for _, community := range communities {+ totalItems += wikiExtractItemCount(community)+ }+ p.planBudget = deriveWikiPlanBudget(p.deps.ModelContextLen, totalItems)+ quotas := allocatePlanQuotas(communities, p.planBudget.Cap())+ approved := wikiExtract{}+ plans := make([]wikiPlan, len(communities))+ jobs := make([]func() error, 0, len(communities))+ for i, community := range communities {+ i, community := i, community+ quota := quotas[i]+ if quota <= 0 {+ continue+ }+ approved.Entities = append(approved.Entities, community.Entities...)+ approved.Concepts = append(approved.Concepts, community.Concepts...)+ approved.Claims = append(approved.Claims, community.Claims...)+ approved.Relations = append(approved.Relations, community.Relations...)+ approved.Topics = append(approved.Topics, community.Topics...)+ jobs = append(jobs, func() error {+ if err := p.ctx.Err(); err != nil {+ return err+ }+ plan, err := p.runPlanBatch(community, i+1, len(communities), quota, p.planBudget.MaxTokens)+ if err != nil {+ return err+ }+ group := fmt.Sprintf("community-%d", i+1)+ for pageIndex := range plan.Pages {+ plan.Pages[pageIndex].PlanGroup = group+ }+ plans[i] = plan+ return nil+ })+ }+ if err := runBatches(p.ctx, jobs); err != nil {+ return wikiPlan{}, err+ }+ merged := p.mergePlanCandidates(plans, approved)+ var excluded int+ merged.Pages, excluded = truncatePlanPagesByCap(merged.Pages, p.planBudget.Max, approved)+ p.planCapacityExcluded += excluded+ merged.Pages = normalizeWikiPlanPageLinks(merged.Pages)+ reconciled, err := p.reconcilePlan(merged)+ if err != nil {+ return wikiPlan{}, err+ }+ reconciled.Pages = normalizeWikiPlanPageLinks(reconciled.Pages)+ appcommon.Info("wiki: topic candidate communities planned",+ zap.String("dataset_id", p.datasetID), zap.String("doc_id", p.runKey()),+ zap.Int("communities", len(communities)), zap.Int("plan_pages", len(reconciled.Pages)))+ return reconciled, nil+}diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex f4acf69110..ebac9d9d4a 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -636,7 +636,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// existing dataset-level merge path below. This is the per-variant dispatch// the plan requires: tree and structure both produce a nav by-product (B2),// so both upsert their summary into the cross-document nav tree; wiki products- // continue to the replace-only merge.+ // continue to the page-specific evidence merge.navIn := navInputFromProducts(kb, incoming)if len(navIn) > 0 {ns := nav.GetNavService()@@ -671,6 +671,13 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// contract. Legacy rows whose kind is empty are derived in the Reader; only// truly page-kind wiki products proceed.candidates := filterWikiPageCandidates(incoming)+ entityMerged, topicCandidates, err := c.mergeEntityModeCandidates(ctx, tenant, kb, candidates)+ if err != nil {+ return err+ }+ // Entity pages have stable page identity and must bypass topic-mode KNN and+ // Page-specific entity/concept merging. Only topic-mode pages continue below.+ candidates = topicCandidates// In-memory dedup among the completed batch first.candidates, dedupErr := deduper.Dedup(ctx, candidates)if dedupErr != nil {@@ -738,6 +745,27 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, eunmatched []kccommon.ProductgroupsMu sync.Mutex)+ // Exact topic recall is independent of page-content KNN. A page can use+ // different wording from the existing topic while still belonging to the+ // same topic, so consult the current dataset pages before running KNN.+ topicPagesByKey := make(map[string]kccommon.Product)+ if pageReader, ok := c.reader.(mergedWikiPageReader); ok {+ existingPages, err := pageReader.LoadMergedWikiPages(ctx, tenant, kb)+ if err != nil {+ return err+ }+ for _, page := range existingPages {+ if !isTopicPage(page) {+ continue+ }+ key := topicKey(productTopic(page))+ if key != "" {+ if _, exists := topicPagesByKey[key]; !exists {+ topicPagesByKey[key] = page+ }+ }+ }+ }// The KNN pass is docengine-bounded (vector search), not CPU-bounded, so we// fan it out across the shared global compilerPool (vCPU-sized). Output order// is irrelevant: merged rows are upserted by their idempotent dataset-level@@ -745,6 +773,19 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ejobs := make([]CompilerJob, 0, len(candidates))for _, cand := range candidates {cand := cand+ if isTopicPage(cand) {+ if existing, ok := topicPagesByKey[topicKey(productTopic(cand))]; ok {+ groupsMu.Lock()+ g := groupsByID[existing.ID]+ if g == nil {+ g = &mergeGroup{existing: existing, score: 1}+ groupsByID[existing.ID] = g+ }+ g.candidates = append(g.candidates, cand)+ groupsMu.Unlock()+ continue+ }+ }jobs = append(jobs, func() error {var vec64 []float64if len(cand.Vector) > 0 {@@ -797,10 +838,59 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ereturn err}+ // KNN only narrows topic-page candidates. When the hit is a topic page,+ // require an explicit topic match or an LLM topic-route decision before+ // allowing the generic Wiki evidence merge to fold the pages together.+ // Otherwise semantically unrelated pages with similar prose would inherit+ // the existing page's topic.+ router, hasRouter := deduper.(topicRouter)+ if hasRouter || len(groupsByID) > 0 {+ for _, group := range groupsByID {+ if !isTopicPage(group.existing) {+ continue+ }+ accepted := make([]kccommon.Product, 0, len(group.candidates))+ for _, candidate := range group.candidates {+ if !isTopicPage(candidate) {+ candidate.Merged = true+ candidate.DocID = kb+ unmatched = append(unmatched, candidate)+ continue+ }+ if topicKey(productTopic(candidate)) == topicKey(productTopic(group.existing)) {+ accepted = append(accepted, candidate)+ continue+ }+ merge, routedTopic := false, ""+ if hasRouter {+ var err error+ merge, routedTopic, err = router.RouteTopic(ctx, candidate, group.existing)+ if err != nil {+ return err+ }+ }+ if !merge {+ candidate.Merged = true+ candidate.DocID = kb+ unmatched = append(unmatched, candidate)+ continue+ }+ if routedTopic == "" {+ routedTopic = productTopic(group.existing)+ }+ candidate = prepareTopicProduct(candidate, routedTopic)+ candidate.Meta["title"] = routedTopic+ accepted = append(accepted, candidate)+ }+ group.candidates = accepted+ }+ }+// Fold every KNN group into the LLM in a single batch round-trip (one// DecideBatch call instead of one Decide per pair), then collect the// updated existing rows and the candidates judged distinct (new rows).var newMerged []kccommon.Product+ newMerged = append(newMerged, entityMerged...)if len(groupsByID) > 0 {// Diagnostics: summarize the KNN groups before the LLM merge decision so// the reader can see which existing merged rows were hit and by how many@@ -851,8 +941,22 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, emergedFinal := make([]kccommon.Product, 0, len(newMerged)+len(unmatched))mergedFinal = append(mergedFinal, newMerged...)mergedFinal = append(mergedFinal, unmatched...)+ mergedFinal, staleTopicIDs := mergeTopicProducts(tenant, kb, mergedFinal)+ mergedFinal = refreshWikiProductVectors(ctx, tenant, mergedFinal)if err := c.withWriteLock(ctx, kb, token, func() error {- return c.writer.WriteMerged(ctx, tenant, kb, mergedFinal)+ if err := c.writer.WriteMerged(ctx, tenant, kb, mergedFinal); err != nil {+ return err+ }+ if len(staleTopicIDs) == 0 {+ return nil+ }+ deleter, ok := c.writer.(interface {+ DeleteMergedWikiPages(context.Context, string, string, []string) error+ })+ if !ok {+ return fmt.Errorf("wiki topic merge requires page deletion support for %d stale page(s)", len(staleTopicIDs))+ }+ return deleter.DeleteMergedWikiPages(ctx, tenant, kb, staleTopicIDs)}); err != nil {if errors.Is(err, errClaimSuperseded) {common.Info("knowledge_compile: batch stale before write, aborting (rewrite barrier)",@@ -895,6 +999,121 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ereturn nil}+func copyMeta(input map[string]any) map[string]any {+ output := make(map[string]any, len(input)+2)+ for key, value := range input {+ output[key] = value+ }+ return output+}++// mergeEntityModeCandidates handles the stage-3 entity page contract. Entity+// and concept pages are keyed by their deterministic slug, not by embedding+// proximity. A batch may contain evidence for the same page from multiple+// documents; fold it before WriteMerged and retain the existing page body when+// it is already present. Topic pages remain on the KNN/LLM route below.+func (c *Consumer) mergeEntityModeCandidates(ctx context.Context, tenant, kb string, candidates []kccommon.Product) ([]kccommon.Product, []kccommon.Product, error) {+ groups := make(map[string][]kccommon.Product)+ var topic []kccommon.Product+ for _, candidate := range candidates {+ pageType := strings.ToLower(strings.TrimSpace(metaString(candidate.Meta, "page_type")))+ if pageType != "entity" && pageType != "concept" {+ topic = append(topic, candidate)+ continue+ }+ key := datasetLevelID(tenant, kb, candidate)+ groups[key] = append(groups[key], candidate)+ }+ if len(groups) == 0 {+ return nil, topic, nil+ }+ keys := make([]string, 0, len(groups))+ for key := range groups {+ keys = append(keys, key)+ }+ sort.Strings(keys)+ reader, canLoadExisting := c.reader.(mergedProductReader)+ deps, depsErr := kccommon.ResolveDeps(tenant, defaultLLMID, defaultEmbedding)+ merged := make([]kccommon.Product, 0, len(keys))+ for _, key := range keys {+ var current kccommon.Product+ if canLoadExisting {+ var err error+ current, err = reader.LoadMergedProduct(ctx, tenant, kb, key)+ if err != nil {+ return nil, nil, err+ }+ }+ for _, candidate := range groups[key] {+ candidate.Merged = true+ candidate.DocID = kb+ if current.ID == "" {+ current = candidate+ } else {+ if depsErr == nil {+ current = c.synthesizeEntityPage(ctx, deps, current, candidate)+ } else {+ current = wikiEntityMerge(current, candidate)+ }+ }+ }+ if current.ID != "" {+ current.Merged = true+ current.DocID = kb+ merged = append(merged, current)+ }+ }+ return merged, topic, nil+}++// synthesizeEntityPage updates one entity page from the complete old and new+// evidence. Unlike topic-mode replacement, the model receives both bodies and+// is required to preserve every factual block. If the model cannot be resolved+// (for example in an offline test), the deterministic block union remains safe.+func (c *Consumer) synthesizeEntityPage(ctx context.Context, deps kccommon.Deps, existing, incoming kccommon.Product) kccommon.Product {+ merged := wikiEntityMerge(existing, incoming)+ if deps.Chat == nil {+ return merged+ }+ resp, err := deps.Chat.Chat(ctx, kccommon.ChatRequest{+ LLMID: defaultLLMID,+ SystemPrompt: "You are an entity wiki editor. Rewrite the complete page from both evidence versions. Preserve every supported factual statement, do not invent facts, and return only Markdown.",+ UserPrompt: fmt.Sprintf("Entity page: %s\n\nEXISTING PAGE:\n%s\n\nNEW EVIDENCE PAGE:\n%s\n\nWrite the complete merged page. Do not omit facts that appear in either version.",+ metaString(existing.Meta, "title"), existing.Content, incoming.Content),+ })+ if err != nil || resp == nil || strings.TrimSpace(resp.Content) == "" {+ return merged+ }+ merged.Content = strings.TrimSpace(resp.Content)+ return merged+}++func refreshWikiProductVectors(ctx context.Context, tenant string, products []kccommon.Product) []kccommon.Product {+ if len(products) == 0 {+ return products+ }+ deps, err := kccommon.ResolveDeps(tenant, defaultLLMID, defaultEmbedding)+ if err != nil || deps.Embed == nil {+ return products+ }+ vectors, err := deps.Embed.Encode(ctx, productContentsForEmbedding(products))+ if err != nil || len(vectors) != len(products) {+ return products+ }+ for i := range products {+ products[i].Vector = vectors[i]+ }+ return products+}++func productContentsForEmbedding(products []kccommon.Product) []string {+ contents := make([]string, len(products))+ for i := range products {+ contents[i] = products[i].Content+ }+ return contents+}+// candidateIdentity returns a compact identity string for a product, preferring// the wiki slug (full "<page_type>/<slug>") when present, else the id/doc_id,// so KNN/dedup diagnostics can tie a candidate to the page it belongs to.diff --git a/internal/ingestion/knowledge_compile/dedup.go b/internal/ingestion/knowledge_compile/dedup.goindex 37785f6481..a35c0c5702 100644--- a/internal/ingestion/knowledge_compile/dedup.go+++ b/internal/ingestion/knowledge_compile/dedup.go@@ -112,11 +112,9 @@ func (x *llmDeduper) Decide(ctx context.Context, existing, incoming kccommon.Pro}// DecideBatch folds every group into its dataset-level merged row. Wiki groups-// use a REPLACE-ONLY strategy (wikiDecideBatch) that never invokes the LLM JSON-// merge — Markdown wiki pages must not be concatenated/merged by the generic-// structure decider. Structure (and other non-wiki) groups are judged by the LLM-// merge decider as before.-// splitWikiGroups partitions the batch into wiki groups (replace-only, no LLM)+// use the page-specific Markdown evidence merge, while structure groups are+// judged by the generic LLM merge decider.+// splitWikiGroups partitions the batch into wiki groups (page merge)// and structure groups (LLM-merged). structIdx[i] is the ORIGINAL position in// `groups` of structGroups[i]; it is what the fold uses to write results back to// the right slice element. Recording the position inside structGroups (i.e.@@ -135,11 +133,16 @@ func splitWikiGroups(groups []MergeGroup) (wikiIdx, structIdx []int, structGroup}func (x *llmDeduper) DecideBatch(ctx context.Context, groups []MergeGroup) ([]MergeGroup, error) {- // Split into wiki (replace-only, no LLM) and structure (LLM-merged) groups.+ // Split wiki and structure groups because Markdown needs a page-specific+ // merge and must not be parsed as generic JSON.wikiIdx, structIdx, structGroups := splitWikiGroups(groups)- // Wiki groups: replace-only, in place.- for _, gi := range wikiIdx {- groups[gi] = wikiDecideBatch(ctx, []MergeGroup{groups[gi]})[0]+ // Wiki groups: evidence-preserving page merge, in place.+ wikiGroups := make([]MergeGroup, len(wikiIdx))+ for i, gi := range wikiIdx {+ wikiGroups[i] = groups[gi]+ }+ for i, group := range wikiMergeBatch(ctx, wikiGroups) {+ groups[wikiIdx[i]] = group}if len(structGroups) == 0 {return groups, nildiff --git a/internal/ingestion/knowledge_compile/nav_dispatch_test.go b/internal/ingestion/knowledge_compile/nav_dispatch_test.goindex 5fdd913fb4..717d385ed3 100644--- a/internal/ingestion/knowledge_compile/nav_dispatch_test.go+++ b/internal/ingestion/knowledge_compile/nav_dispatch_test.go@@ -60,7 +60,7 @@ func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {}// TestNavInputFromProducts_WikiExcluded covers the dispatch boundary: wiki-// products never become nav inputs (they go to the replace-only wiki merge).+// products never become nav inputs (they go to the page-specific wiki merge).func TestNavInputFromProducts_WikiExcluded(t *testing.T) {products := []kccommon.Product{{DocID: "d1", TenantID: "t1", Variant: kccommon.VariantWiki,diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 7a75e52c96..fb3b6364b3 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -53,6 +53,18 @@ type Reader interface {SearchSimilar(ctx context.Context, tenant, kb string, variant kccommon.Variant, vector []float64, topN int, minScore float64) (kccommon.Product, float64, error)}+// mergedProductReader is an optional extension used by entity-mode Wiki. It+// loads one stable dataset-level page by id; keeping it separate from Reader+// avoids forcing test/offline readers and non-Wiki variants to implement a+// dataset-row lookup they do not need.+type mergedProductReader interface {+ LoadMergedProduct(ctx context.Context, tenant, kb, id string) (kccommon.Product, error)+}++type mergedWikiPageReader interface {+ LoadMergedWikiPages(ctx context.Context, tenant, kb string) ([]kccommon.Product, error)+}+// engineReader loads the per-document compiled products through the global// DocEngine (§11.6 step 1, §11.7 incremental re-dedup). It depends on the// process-wide DocEngine obtained via engine.Get(); the engine abstraction owns@@ -103,7 +115,7 @@ var compiledSelectFields = []string{// graph keeps accumulating cross-run duplicates. ES accepts the wildcard in// both _source includes and the fields parameter.var wikiSelectFields = []string{- "page_type_kwd", "topic_kwd", "title_kwd",+ "page_type_kwd", "topic_kwd", "plan_group_kwd", "generation_kwd", "title_kwd","entity_names_kwd", "summary_with_weight","related_kb_pages_kwd", "outlinks_kwd", "section_level_int","q_*_vec",@@ -181,6 +193,69 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID strreturn out, nil}+func (r engineReader) LoadMergedProduct(ctx context.Context, tenant, kb, id string) (kccommon.Product, error) {+ eng := r.eng+ if eng == nil {+ eng = engine.Get()+ }+ if eng == nil || id == "" {+ return kccommon.Product{}, nil+ }+ filter := map[string]interface{}{"id": id, "available_int": 1, "scope_kwd": "dataset"}+ res, err := eng.Search(ctx, &types.SearchRequest{+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)}, KbIDs: []string{kb}, Limit: 1,+ SelectFields: append(append([]string(nil), compiledSelectFields...), wikiSelectFields...),+ Filter: filter,+ })+ if err != nil {+ return kccommon.Product{}, err+ }+ if res == nil || len(res.Chunks) == 0 {+ return kccommon.Product{}, nil+ }+ product, ok := productFromChunkMap(res.Chunks[0], tenant, kccommon.VariantWiki)+ if !ok {+ return kccommon.Product{}, nil+ }+ return product, nil+}++func (r engineReader) LoadMergedWikiPages(ctx context.Context, tenant, kb string) ([]kccommon.Product, error) {+ eng := r.eng+ if eng == nil {+ eng = engine.Get()+ }+ if eng == nil {+ return nil, nil+ }+ filter := map[string]interface{}{"available_int": 1, "scope_kwd": "dataset", "compile_kwd": compileKwdWikiPage}+ var pages []kccommon.Product+ for offset := 0; ; offset += loadDocProductsLimit {+ res, err := eng.Search(ctx, &types.SearchRequest{+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)}, KbIDs: []string{kb},+ Limit: loadDocProductsLimit, Offset: offset,+ OrderBy: (&types.OrderByExpr{}).Asc("id"),+ SelectFields: append(append([]string(nil), compiledSelectFields...), wikiSelectFields...),+ Filter: filter,+ })+ if err != nil {+ return nil, err+ }+ if res == nil || len(res.Chunks) == 0 {+ break+ }+ for _, row := range res.Chunks {+ if product, ok := productFromChunkMap(row, tenant, kccommon.VariantWiki); ok && metaString(product.Meta, "kind") == "page" {+ pages = append(pages, product)+ }+ }+ if len(res.Chunks) < loadDocProductsLimit {+ break+ }+ }+ return pages, nil+}+// productFromChunkMap reconstructs a kccommon.Product from a stored compiled// chunk document. It reads the payload from kc_payload (falling back to// content_with_weight) and the embedding from the q_<dim>_vec column.@@ -307,8 +382,14 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo} else if variant == compileKwdWikiSection {meta["kind"] = "section"}+ if v := asString(c["plan_group_kwd"]); v != "" {+ meta["plan_group"] = v+ }+ if v := asString(c["generation_kwd"]); v != "" {+ meta["generation"] = v+ }// wiki_incremental port: restore the original creation timestamp so a- // replace-only merge (wikiMerge.Replace) preserves it instead of stamping+ // page merge preserves it instead of stamping// a fresh now(). existing rows carry create_timestamp_flt (and optionally// a human-readable create_time string).if v, ok := metaFloat(c, "create_timestamp_flt"); ok {diff --git a/internal/ingestion/knowledge_compile/scheduler_test.go b/internal/ingestion/knowledge_compile/scheduler_test.goindex dc413dee21..6f749cbcae 100644--- a/internal/ingestion/knowledge_compile/scheduler_test.go+++ b/internal/ingestion/knowledge_compile/scheduler_test.go@@ -2,8 +2,10 @@ package knowledge_compileimport ("encoding/json"+ "errors""reflect""testing"+ "time")// TestRemoveEntriesMatchesByDocAndEventType locks the A0-1 contract: BacklogEntry@@ -32,6 +34,54 @@ func TestRemoveEntriesMatchesByDocAndEventType(t *testing.T) {}}+func TestFakeSchedulerReclaimsInterruptedClaim(t *testing.T) {+ f := NewFakeScheduler()+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}); err != nil {+ t.Fatalf("publish: %v", err)+ }+ first, ok, err := f.Claim(t.Context(), "kb1")+ if err != nil || !ok {+ t.Fatalf("initial claim: ok=%v err=%v", ok, err)+ }+ f.mu.Lock()+ expired := time.Now().Add(-time.Second)+ f.rows["kb1"].expires = &expired+ f.mu.Unlock()+ second, ok, err := f.TryClaim(t.Context())+ if err != nil || !ok {+ t.Fatalf("reclaimed claim: ok=%v err=%v", ok, err)+ }+ if second.Token == first.Token || len(second.Entries) != 1 || second.Entries[0].DocID != "d1" {+ t.Fatalf("reclaimed claim did not replace the old lease: first=%+v second=%+v", first, second)+ }+}++func TestWithWriteLockRejectsSupersededClaim(t *testing.T) {+ f := NewFakeScheduler()+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"wiki"}); err != nil {+ t.Fatalf("publish: %v", err)+ }+ claim, ok, err := f.Claim(t.Context(), "kb1")+ if err != nil || !ok {+ t.Fatalf("claim: ok=%v err=%v", ok, err)+ }+ c := &Consumer{scheduler: f}+ if err := f.CancelInflight(t.Context(), "kb1", claim.Token); err != nil {+ t.Fatalf("cancel inflight: %v", err)+ }+ writes := 0+ err = c.withWriteLock(t.Context(), "kb1", claim.Token, func() error {+ writes+++ return nil+ })+ if !errors.Is(err, errClaimSuperseded) {+ t.Fatalf("withWriteLock error = %v, want errClaimSuperseded", err)+ }+ if writes != 0 {+ t.Fatalf("superseded claim executed a write callback")+ }+}+// TestRemoveEntriesEmptyBatch verifies removeEntries with an empty batch is a no-op.func TestRemoveEntriesEmptyBatch(t *testing.T) {inflight := []BacklogEntry{{DocID: "d1", EventType: "doc_completed", Variants: []string{"tree"}}}diff --git a/internal/ingestion/knowledge_compile/topic_merge.go b/internal/ingestion/knowledge_compile/topic_merge.gonew file mode 100644index 0000000000..e9e0899ef9--- /dev/null+++ b/internal/ingestion/knowledge_compile/topic_merge.go@@ -0,0 +1,172 @@+package knowledge_compile++import (+ "context"+ "fmt"+ "strings"++ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++const topicRouteSystemPrompt = "You route incoming Wiki pages to an existing knowledge-base topic. Decide by semantic topic relevance, not title similarity alone. Return only JSON: {\"merge\":true|false,\"topic\":\"canonical topic label\"}. Set merge=true only when the incoming page belongs to the candidate topic. Set merge=false when it should remain a separate topic."++type topicRouter interface {+ RouteTopic(ctx context.Context, incoming, existing kccommon.Product) (bool, string, error)+}++func (x *llmDeduper) RouteTopic(ctx context.Context, incoming, existing kccommon.Product) (bool, string, error) {+ if x == nil || x.decider == nil || x.decider.Chat == nil {+ return false, "", nil+ }+ prompt := fmt.Sprintf("INCOMING PAGE\ntopic: %s\ntitle: %s\nentities: %s\nsummary: %s\n\nCANDIDATE TOPIC PAGE\ntopic: %s\ntitle: %s\nentities: %s\nsummary: %s",+ productTopic(incoming), productTitle(incoming), strings.Join(productEntities(incoming), ", "), productSummary(incoming),+ productTopic(existing), productTitle(existing), strings.Join(productEntities(existing), ", "), productSummary(existing))+ result, err := kccommon.GenJSON(ctx, x.decider.Chat, kccommon.ChatRequest{+ LLMID: x.decider.LLMID, SystemPrompt: topicRouteSystemPrompt, UserPrompt: prompt,+ })+ if err != nil {+ return false, "", err+ }+ merge, _ := result["merge"].(bool)+ topic, _ := result["topic"].(string)+ return merge, strings.TrimSpace(topic), nil+}++func isTopicPage(product kccommon.Product) bool {+ return strings.EqualFold(strings.TrimSpace(metaString(product.Meta, "page_type")), "topic")+}++func productTopic(product kccommon.Product) string {+ return strings.TrimSpace(metaString(product.Meta, "topic"))+}++func productTitle(product kccommon.Product) string {+ return strings.TrimSpace(metaString(product.Meta, "title"))+}++func productSummary(product kccommon.Product) string {+ if summary := strings.TrimSpace(metaString(product.Meta, "summary")); summary != "" {+ return summary+ }+ return strings.TrimSpace(product.Content)+}++func productEntities(product kccommon.Product) []string {+ return metaStringSlice(product.Meta, "entity_names")+}++func firstTopicString(values ...string) string {+ for _, value := range values {+ if value = strings.TrimSpace(value); value != "" {+ return value+ }+ }+ return ""+}++func topicKey(topic string) string {+ return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(topic)), " "))+}++func topicSlug(topic string) string {+ return "topic/" + hashStr(topicKey(topic))+}++func prepareTopicProduct(product kccommon.Product, topic string) kccommon.Product {+ product.Meta = copyMeta(product.Meta)+ topic = strings.TrimSpace(topic)+ if topic == "" {+ topic = productTopic(product)+ }+ product.Meta["page_type"] = "topic"+ product.Meta["topic"] = topic+ product.Meta["slug"] = topicSlug(topic)+ if productTitle(product) == "" {+ product.Meta["title"] = topic+ }+ return product+}++// mergeTopicProducts folds pages with the same canonical topic and preserves+// the first dataset-level page identity.+func mergeTopicProducts(tenant, kb string, products []kccommon.Product) ([]kccommon.Product, []string) {+ groups := make(map[string][]kccommon.Product)+ order := make([]string, 0, len(products))+ for _, product := range products {+ key := candidateIdentity(product)+ if isTopicPage(product) {+ key = topicKey(productTopic(product))+ if key == "" {+ key = candidateIdentity(product)+ }+ }+ if _, exists := groups[key]; !exists {+ order = append(order, key)+ }+ groups[key] = append(groups[key], product)+ }+ merged := make([]kccommon.Product, 0, len(groups))+ stale := make([]string, 0)+ for _, key := range order {+ items := groups[key]+ if len(items) == 0 {+ continue+ }+ current := items[0]+ if isTopicPage(current) {+ existingSlug := metaString(current.Meta, "slug")+ current = prepareTopicProduct(current, productTopic(current))+ if existingSlug != "" && current.Merged && current.DocID == kb {+ current.Meta["slug"] = existingSlug+ }+ }+ for _, item := range items[1:] {+ if item.ID != "" && item.ID != current.ID && item.Merged && item.DocID == kb {+ stale = append(stale, item.ID)+ }+ if isTopicPage(current) {+ item = prepareTopicProduct(item, productTopic(current))+ current = mergeTopicPage(current, item)+ } else {+ current = wikiEntityMerge(current, item)+ }+ }+ current.Merged = true+ currentID := datasetLevelID(tenant, kb, current)+ filteredStale := stale[:0]+ for _, id := range stale {+ if id != currentID {+ filteredStale = append(filteredStale, id)+ }+ }+ stale = filteredStale+ merged = append(merged, current)+ }+ // The writer's dataset-level id is derived from the surviving topic slug.+ // Remove superseded dataset rows after the replacement is written.+ seen := make(map[string]struct{}, len(stale))+ uniqueStale := make([]string, 0, len(stale))+ for _, id := range stale {+ if id == "" {+ continue+ }+ if _, exists := seen[id]; exists {+ continue+ }+ seen[id] = struct{}{}+ uniqueStale = append(uniqueStale, id)+ }+ return merged, uniqueStale+}++func mergeTopicPage(existing, incoming kccommon.Product) kccommon.Product {+ merged := wikiEntityMerge(existing, incoming)+ merged.Meta = copyMeta(merged.Meta)+ merged.Meta["slug"] = metaString(existing.Meta, "slug")+ merged.Meta["page_type"] = "topic"+ topic := firstTopicString(metaString(incoming.Meta, "topic"), metaString(existing.Meta, "topic"))+ merged.Meta["topic"] = topic+ merged.Meta["title"] = firstTopicString(metaString(incoming.Meta, "title"), metaString(existing.Meta, "title"), topic)+ merged.Meta["entity_names"] = unionStrs(metaStringSlice(existing.Meta, "entity_names"), metaStringSlice(incoming.Meta, "entity_names"))+ return merged+}diff --git a/internal/ingestion/knowledge_compile/topic_merge_test.go b/internal/ingestion/knowledge_compile/topic_merge_test.gonew file mode 100644index 0000000000..92a26d610a--- /dev/null+++ b/internal/ingestion/knowledge_compile/topic_merge_test.go@@ -0,0 +1,71 @@+package knowledge_compile++import (+ "strings"+ "testing"++ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++func TestMergeTopicProductsPreservesEvidenceAndStableTopicIdentity(t *testing.T) {+ products := []kccommon.Product{+ {+ ID: "doc-page-1", DocID: "doc-1", Variant: kccommon.VariantWiki, Content: "# History\n\nfirst",+ Meta: map[string]any{+ "kind": "page", "page_type": "topic", "topic": "History", "title": "History",+ "slug": "topic/history-a", "entity_names": []string{"Alice"},+ "source_doc_ids": []string{"doc-1"}, "source_chunk_ids": []string{"chunk-1"},+ },+ },+ {+ ID: "doc-page-2", DocID: "doc-2", Variant: kccommon.VariantWiki, Content: "# History\n\nsecond",+ Meta: map[string]any{+ "kind": "page", "page_type": "topic", "topic": " history ", "title": "History",+ "slug": "topic/history-b", "entity_names": []string{"Bob"},+ "source_doc_ids": []string{"doc-2"}, "source_chunk_ids": []string{"chunk-2"},+ },+ },+ }+ merged, stale := mergeTopicProducts("tenant", "kb", products)+ if len(merged) != 1 {+ t.Fatalf("merged pages = %d, want 1", len(merged))+ }+ if len(stale) != 0 {+ t.Fatalf("stale dataset pages = %v, want none for document products", stale)+ }+ if !strings.Contains(merged[0].Content, "first") || !strings.Contains(merged[0].Content, "second") {+ t.Fatalf("merged content lost evidence: %q", merged[0].Content)+ }+ if got := metaString(merged[0].Meta, "slug"); got != "topic/"+hashStr("history") {+ t.Fatalf("topic slug = %q, want deterministic topic slug", got)+ }+ if got := metaStringSlice(merged[0].Meta, "entity_names"); len(got) != 2 {+ t.Fatalf("entity names = %v, want both entities", got)+ }+}++func TestMergeTopicProductsKeepsDifferentTopicsSeparate(t *testing.T) {+ products := []kccommon.Product{+ {ID: "a", Variant: kccommon.VariantWiki, Content: "a", Meta: map[string]any{"kind": "page", "page_type": "topic", "topic": "History"}},+ {ID: "b", Variant: kccommon.VariantWiki, Content: "b", Meta: map[string]any{"kind": "page", "page_type": "topic", "topic": "Science"}},+ }+ merged, _ := mergeTopicProducts("tenant", "kb", products)+ if len(merged) != 2 {+ t.Fatalf("merged pages = %d, want 2", len(merged))+ }+}++func TestMergeTopicProductsRemovesSupersededDatasetPage(t *testing.T) {+ products := []kccommon.Product{+ {ID: "old-page", DocID: "kb", Merged: true, Variant: kccommon.VariantWiki, Content: "old", Meta: map[string]any{+ "kind": "page", "page_type": "topic", "topic": "History", "slug": "topic/old",+ }},+ {ID: "new-page", DocID: "kb", Merged: true, Variant: kccommon.VariantWiki, Content: "new", Meta: map[string]any{+ "kind": "page", "page_type": "topic", "topic": "History", "slug": "topic/new",+ }},+ }+ merged, stale := mergeTopicProducts("tenant", "kb", products)+ if len(merged) != 1 || len(stale) != 1 || stale[0] != "new-page" {+ t.Fatalf("merged=%#v stale=%#v, want one survivor and new-page stale", merged, stale)+ }+}diff --git a/internal/ingestion/knowledge_compile/wiki_map_store.go b/internal/ingestion/knowledge_compile/wiki_map_store.gonew file mode 100644index 0000000000..88e40985e0--- /dev/null+++ b/internal/ingestion/knowledge_compile/wiki_map_store.go@@ -0,0 +1,180 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package knowledge_compile++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "strings"++ "ragflow/internal/engine"+ "ragflow/internal/engine/types"+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++const (+ wikiMapExtractCompileKWD = "wiki_map_extract"+ wikiMapStoreBatchSize = 500+)++type wikiMapVersionStore struct {+ engine engine.DocEngine+}++// NewWikiMapVersionStore returns the DocStore-backed immutable Wiki MAP cache.+// Its rows remain non-searchable through available_int=0 and the compile+// discriminator, while preserving every chunk/hash version for reuse.+func NewWikiMapVersionStore(docEngine engine.DocEngine) kccommon.WikiMapVersionStore {+ return &wikiMapVersionStore{engine: docEngine}+}++func (s *wikiMapVersionStore) GetWikiMapVersions(ctx context.Context, tenantID, datasetID string, keys []string) (map[string][]byte, error) {+ out := make(map[string][]byte, len(keys))+ if len(keys) == 0 {+ return out, nil+ }+ if s == nil || s.engine == nil {+ return nil, fmt.Errorf("wiki MAP version DocStore is not initialized")+ }+ if strings.TrimSpace(tenantID) == "" || strings.TrimSpace(datasetID) == "" {+ return nil, fmt.Errorf("load Wiki MAP versions: tenant_id and dataset_id are required")+ }+ for start := 0; start < len(keys); start += wikiMapStoreBatchSize {+ end := min(start+wikiMapStoreBatchSize, len(keys))+ result, err := s.engine.Search(ctx, &types.SearchRequest{+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},+ KbIDs: []string{datasetID},+ Limit: end - start,+ SelectFields: []string{+ "id", "compile_kwd", "content_with_weight",+ },+ Filter: map[string]interface{}{+ "id": keys[start:end],+ "compile_kwd": wikiMapExtractCompileKWD,+ "available_int": 0,+ },+ })+ if err != nil {+ return nil, fmt.Errorf("load Wiki MAP versions: %w", err)+ }+ if result == nil {+ continue+ }+ for _, row := range result.Chunks {+ if mapStoreString(row["compile_kwd"]) != wikiMapExtractCompileKWD {+ continue+ }+ id := mapStoreString(row["id"])+ payload := mapStoreString(row["content_with_weight"])+ if id != "" && payload != "" {+ out[id] = []byte(payload)+ }+ }+ }+ return out, nil+}++func (s *wikiMapVersionStore) PutWikiMapVersions(ctx context.Context, versions []kccommon.WikiMapVersion) error {+ if len(versions) == 0 {+ return nil+ }+ if s == nil || s.engine == nil {+ return fmt.Errorf("wiki MAP version DocStore is not initialized")+ }+ byScope := make(map[string][]kccommon.WikiMapVersion)+ for _, version := range versions {+ if version.Key == "" || version.DatasetID == "" || version.TenantID == "" {+ return fmt.Errorf("save Wiki MAP version: key, tenant_id, and dataset_id are required")+ }+ scope := version.TenantID + "\x00" + version.DatasetID+ byScope[scope] = append(byScope[scope], version)+ }+ for _, scopedVersions := range byScope {+ for start := 0; start < len(scopedVersions); start += wikiMapStoreBatchSize {+ end := min(start+wikiMapStoreBatchSize, len(scopedVersions))+ batch := scopedVersions[start:end]+ keys := make([]string, len(batch))+ for i := range batch {+ keys[i] = batch[i].Key+ }+ existing, err := s.GetWikiMapVersions(ctx, batch[0].TenantID, batch[0].DatasetID, keys)+ if err != nil {+ return err+ }+ rows := make([]map[string]interface{}, 0, len(batch))+ for _, version := range batch {+ if _, exists := existing[version.Key]; !exists {+ rows = append(rows, wikiMapVersionRow(version))+ }+ }+ if len(rows) == 0 {+ continue+ }+ if _, err := s.engine.InsertChunks(ctx, rows, fmt.Sprintf("ragflow_%s", batch[0].TenantID), batch[0].DatasetID); err != nil {+ return fmt.Errorf("save Wiki MAP versions: %w", err)+ }+ }+ }+ return nil+}++func wikiMapVersionRow(version kccommon.WikiMapVersion) map[string]interface{} {+ return map[string]interface{}{+ "id": version.Key,+ // Keep immutable MAP history in a separate document namespace so source+ // document deletion cannot remove a reusable chunk/hash version.+ "doc_id": wikiMapCacheDocID(version.DocumentID),+ "tenant_id": version.TenantID,+ "kb_id": version.DatasetID,+ "compile_kwd": wikiMapExtractCompileKWD,+ "scope_kwd": "doc",+ "source_chunk_ids": []string{version.ChunkID},+ "source_doc_ids": []string{version.DocumentID},+ "chunk_hash_kwd": version.ContentHash,+ "input_hash_kwd": wikiMapInputFingerprint(version),+ "content_with_weight": string(version.Payload),+ "available_int": 0,+ }+}++func wikiMapCacheDocID(documentID string) string {+ return "wiki_map_cache:" + documentID+}++func wikiMapInputFingerprint(version kccommon.WikiMapVersion) string {+ sum := sha256.Sum256([]byte(version.TemplateFingerprint + "\x00" + version.LLMFingerprint))+ return hex.EncodeToString(sum[:])+}++func mapStoreString(value interface{}) string {+ switch typed := value.(type) {+ case string:+ return strings.TrimSpace(typed)+ case []string:+ if len(typed) == 1 {+ return strings.TrimSpace(typed[0])+ }+ case []interface{}:+ if len(typed) == 1 {+ if value, ok := typed[0].(string); ok {+ return strings.TrimSpace(value)+ }+ }+ }+ return ""+}diff --git a/internal/ingestion/knowledge_compile/wiki_map_store_test.go b/internal/ingestion/knowledge_compile/wiki_map_store_test.gonew file mode 100644index 0000000000..4d686817e8--- /dev/null+++ b/internal/ingestion/knowledge_compile/wiki_map_store_test.go@@ -0,0 +1,160 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package knowledge_compile++import (+ "context"+ "reflect"+ "testing"++ "ragflow/internal/engine/types"+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++type wikiMapStoreEngine struct {+ fakeEngine+ rows map[string]map[string]interface{}+ inserted []map[string]interface{}+ insertBase string+ insertDataset string+}++func (e *wikiMapStoreEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ e.lastSearchReq = req+ var ids []string+ switch value := req.Filter["id"].(type) {+ case []string:+ ids = value+ case string:+ ids = []string{value}+ }+ if len(ids) == 0 {+ for id := range e.rows {+ ids = append(ids, id)+ }+ }+ chunks := make([]map[string]interface{}, 0, len(ids))+ for _, id := range ids {+ if row, ok := e.rows[id]; ok && rowMatchesFilter(row, req.Filter) {+ chunks = append(chunks, row)+ }+ }+ return &types.SearchResult{Chunks: chunks, Total: int64(len(chunks))}, nil+}++func rowMatchesFilter(row map[string]interface{}, filter map[string]interface{}) bool {+ if expected, ok := filter["available_int"].(int); ok {+ if value, ok := row["available_int"].(int); !ok || value != expected {+ return false+ }+ }+ if mustNot, ok := filter["must_not"].(map[string]interface{}); ok {+ if exists, ok := mustNot["exists"].(string); ok {+ if _, present := row[exists]; present {+ return false+ }+ }+ }+ for _, field := range []string{"compile_kwd", "type_kwd"} {+ if expected, ok := filter[field].(string); ok && mapStoreString(row[field]) != expected {+ return false+ }+ }+ return true+}++func (e *wikiMapStoreEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName, datasetID string) ([]string, error) {+ e.insertBase = baseName+ e.insertDataset = datasetID+ for _, chunk := range chunks {+ copyOfChunk := make(map[string]interface{}, len(chunk))+ for key, value := range chunk {+ copyOfChunk[key] = value+ }+ e.inserted = append(e.inserted, copyOfChunk)+ if e.rows == nil {+ e.rows = map[string]map[string]interface{}{}+ }+ e.rows[mapStoreString(chunk["id"])] = copyOfChunk+ }+ return nil, nil+}++func TestWikiMapVersionStoreUsesNonSearchableDocStoreRows(t *testing.T) {+ engine := &wikiMapStoreEngine{rows: map[string]map[string]interface{}{}}+ store := NewWikiMapVersionStore(engine)+ version := kccommon.WikiMapVersion{+ Key: "version-a",+ TenantID: "tenant-1",+ DatasetID: "kb-1",+ DocumentID: "doc-1",+ ChunkID: "chunk-1",+ ContentHash: "hash-a",+ TemplateFingerprint: "template-a",+ LLMFingerprint: "llm-a",+ Payload: []byte(`{"topics":["original"]}`),+ }+ if err := store.PutWikiMapVersions(t.Context(), []kccommon.WikiMapVersion{version}); err != nil {+ t.Fatalf("PutWikiMapVersions() error = %v", err)+ }+ if len(engine.inserted) != 1 {+ t.Fatalf("inserted rows = %d, want 1", len(engine.inserted))+ }+ row := engine.inserted[0]+ for key, want := range map[string]interface{}{+ "id": "version-a",+ "doc_id": "wiki_map_cache:doc-1",+ "kb_id": "kb-1",+ "compile_kwd": wikiMapExtractCompileKWD,+ "available_int": 0,+ "chunk_hash_kwd": "hash-a",+ } {+ if got := row[key]; !reflect.DeepEqual(got, want) {+ t.Errorf("row[%q] = %#v, want %#v", key, got, want)+ }+ }+ if got := row["source_chunk_ids"]; !reflect.DeepEqual(got, []string{"chunk-1"}) {+ t.Errorf("source_chunk_ids = %#v", got)+ }+ if got := row["source_doc_ids"]; !reflect.DeepEqual(got, []string{"doc-1"}) {+ t.Errorf("source_doc_ids = %#v", got)+ }+ if _, exists := row["q_1024_vec"]; exists {+ t.Fatal("MAP cache row must not carry an embedding")+ }+ if engine.insertBase != "ragflow_tenant-1" || engine.insertDataset != "kb-1" {+ t.Fatalf("insert scope = %q/%q", engine.insertBase, engine.insertDataset)+ }++ version.Payload = []byte(`{"topics":["replacement"]}`)+ if err := store.PutWikiMapVersions(t.Context(), []kccommon.WikiMapVersion{version}); err != nil {+ t.Fatalf("duplicate PutWikiMapVersions() error = %v", err)+ }+ if len(engine.inserted) != 1 {+ t.Fatalf("immutable duplicate inserted; rows = %d, want 1", len(engine.inserted))+ }++ got, err := store.GetWikiMapVersions(t.Context(), "tenant-1", "kb-1", []string{"version-a", "missing"})+ if err != nil {+ t.Fatalf("GetWikiMapVersions() error = %v", err)+ }+ if string(got["version-a"]) != `{"topics":["original"]}` {+ t.Fatalf("stored payload = %s, want immutable original", got["version-a"])+ }+ if engine.lastSearchReq == nil || !reflect.DeepEqual(engine.lastSearchReq.KbIDs, []string{"kb-1"}) {+ t.Fatalf("search KbIDs = %#v", engine.lastSearchReq)+ }+}diff --git a/internal/ingestion/knowledge_compile/wiki_merge.go b/internal/ingestion/knowledge_compile/wiki_merge.goindex f86e2b0cc3..14a89f17c9 100644--- a/internal/ingestion/knowledge_compile/wiki_merge.go+++ b/internal/ingestion/knowledge_compile/wiki_merge.go@@ -17,45 +17,91 @@ package knowledge_compileimport ("context"+ "strings"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common")-// wikiReplace folds a wiki page candidate into its existing dataset-level merged-// row using a REPLACE-ONLY strategy. Wiki pages are Markdown, so the generic-// structure.LLMMergeDecider JSON merge (which concatenates/merges arbitrary JSON-// payloads) would mangle the page body — it must never run on wiki content.-//-// The existing row keeps its identity (id / doc_id / creation time) and provenance-// is unioned (source_doc_ids + source_chunk_ids of existing ∪ candidate). The-// page body is replaced by the candidate's Markdown verbatim. This mirrors-// Python's wiki incremental mode where a page re-compiled from a newer document-// simply supersedes the stored page of the same slug, rather than being merged.-func wikiReplace(existing, candidate kccommon.Product) kccommon.Product {- merged := existing+const maxMergedWikiMarkdownBytes = 256 * 1024- // Body: the newer (incoming) Markdown wins verbatim.- merged.Content = candidate.Content- // The embedding must match the replacement Markdown. WriteMerged persists- // p.Vector without re-embedding, so keeping existing.Vector would leave a- // stale embedding for the old page body and break KNN similarity searches.- merged.Vector = candidate.Vector-- // Identity preserved: keep existing id, doc_id (== kb), and the original- // creation timestamp (carried via Meta.created_at_unix by the Reader).+// wikiEntityMerge combines evidence for the same stable page without+// using a page replacement consumer. Existing and incoming Markdown blocks are+// retained deterministically; source provenance is unioned.+func wikiEntityMerge(existing, incoming kccommon.Product) kccommon.Product {+ if existing.ID == "" {+ incoming.Merged = true+ return incoming+ }+ merged := existing+ merged.Content = unionWikiMarkdown(existing.Content, incoming.Content)+ merged.Vector = incoming.Vector+ merged.Meta = unionWikiProvenance(existing.Meta, incoming.Meta)merged.ID = existing.IDmerged.DocID = existing.DocID+ merged.Merged = true+ return merged+}- // Provenance union (deduped) so the merged page references every source doc- // and chunk that contributed to it across runs.- merged.Meta = unionWikiProvenance(existing.Meta, candidate.Meta)+func unionWikiMarkdown(left, right string) string {+ left, right = strings.TrimSpace(left), strings.TrimSpace(right)+ if left == "" {+ return right+ }+ if right == "" || left == right || strings.Contains(left, right) {+ return left+ }+ if strings.Contains(right, left) {+ return right+ }+ blocks := append(splitMarkdownBlocks(left), splitMarkdownBlocks(right)...)+ seen := map[string]struct{}{}+ unique := make([]string, 0, len(blocks))+ length := 0+ for _, block := range blocks {+ key := strings.TrimSpace(block)+ if key == "" {+ continue+ }+ if _, exists := seen[key]; exists {+ continue+ }+ if length+len(block)+2 > maxMergedWikiMarkdownBytes {+ break+ }+ seen[key] = struct{}{}+ unique = append(unique, key)+ length += len(block) + 2+ }+ return strings.Join(unique, "\n\n")+}- // Re-stamp the resolver/run id from the incoming candidate so the row is- // attributed to the latest batch, but the created_at_unix stays with existing.- if v, ok := candidate.Meta["run_id"]; ok {- merged.Meta["run_id"] = v+func splitMarkdownBlocks(markdown string) []string {+ lines := strings.Split(strings.ReplaceAll(markdown, "\r\n", "\n"), "\n")+ blocks := make([]string, 0)+ var current strings.Builder+ inFence := false+ flush := func() {+ if block := strings.TrimSpace(current.String()); block != "" {+ blocks = append(blocks, block)+ }+ current.Reset()}- return merged+ for _, line := range lines {+ trimmed := strings.TrimSpace(line)+ if strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~") {+ inFence = !inFence+ }+ if trimmed == "" && !inFence {+ flush()+ continue+ }+ if current.Len() > 0 {+ current.WriteByte('\n')+ }+ current.WriteString(line)+ }+ flush()+ return blocks}// unionWikiProvenance returns a new Meta map based on a (the existing row). The@@ -126,26 +172,25 @@ func metaStringSliceAny(m map[string]any, key string) []string {return nil}-// isWikiGroup reports whether a merge group targets the wiki variant and must use-// replace-only (never the JSON-merge decider).+// isWikiGroup reports whether a merge group targets the wiki variant and needs+// the page-specific Markdown merge instead of the generic JSON decider.func isWikiGroup(g MergeGroup) bool {return g.Existing.Variant == kccommon.VariantWiki}-// wikiDecideBatch folds every wiki-group candidate into its existing row using-// replace-only semantics, WITHOUT any LLM call. It mutates the passed groups in-// place and returns them.-func wikiDecideBatch(_ context.Context, groups []MergeGroup) []MergeGroup {+// wikiMergeBatch folds every wiki-group candidate into its existing row while+// retaining Markdown evidence and source provenance.+func wikiMergeBatch(_ context.Context, groups []MergeGroup) []MergeGroup {for gi := range groups {existing := groups[gi].Existingvar distinct []kccommon.Productduplicated := falsefor _, cand := range groups[gi].Candidates {- // A wiki page candidate is always a replacement of the existing row- // (same slug, newer content). It is never kept as an additional distinct- // row — distinctness for wiki is decided by KNN (different slug -> different- // existing row -> different group).- existing = wikiReplace(existing, cand)+ if isTopicPage(existing) && isTopicPage(cand) {+ existing = mergeTopicPage(existing, cand)+ } else {+ existing = wikiEntityMerge(existing, cand)+ }duplicated = true}groups[gi].Merged = existingdiff --git a/internal/ingestion/knowledge_compile/wiki_merge_test.go b/internal/ingestion/knowledge_compile/wiki_merge_test.goindex 75ae2aa08c..4c9cccf8f8 100644--- a/internal/ingestion/knowledge_compile/wiki_merge_test.go+++ b/internal/ingestion/knowledge_compile/wiki_merge_test.go@@ -17,17 +17,16 @@ package knowledge_compileimport ("context"+ "strings""testing"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common")-// TestWikiReplaceCopiesCandidateVector locks the wiki replace-only merge contract-// (CodeRabbit Major): wikiReplace swaps the page body for the candidate's Markdown-// AND must copy the candidate's embedding too. WriteMerged persists Product.Vector-// without re-embedding, so keeping the existing row's vector would leave a stale-// embedding for the old page body and break KNN similarity searches.-func TestWikiReplaceCopiesCandidateVector(t *testing.T) {+// TestWikiMergePreservesCandidateVector verifies that page evidence merging+// retains both Markdown blocks and uses the candidate embedding for the merged+// body.+func TestWikiMergePreservesCandidateVector(t *testing.T) {existing := kccommon.Product{ID: "kb/wiki/alpha",DocID: "kb",@@ -55,14 +54,14 @@ func TestWikiReplaceCopiesCandidateVector(t *testing.T) {},}- groups := wikiDecideBatch(context.Background(), []MergeGroup{{+ groups := wikiMergeBatch(context.Background(), []MergeGroup{{Existing: existing,Candidates: []kccommon.Product{candidate},}})merged := groups[0].Merged- if merged.Content != "# new markdown" {- t.Fatalf("content = %q, want candidate markdown", merged.Content)+ if !strings.Contains(merged.Content, "# old markdown") || !strings.Contains(merged.Content, "# new markdown") {+ t.Fatalf("content = %q, want both evidence blocks", merged.Content)}if !groups[0].Duplicate {t.Fatalf("wiki candidate must be marked duplicate (replacement)")@@ -83,7 +82,37 @@ func TestWikiReplaceCopiesCandidateVector(t *testing.T) {if merged.Meta["created_at"] != "2024-01-01" {t.Fatalf("creation time must be preserved from the existing row, got %v", merged.Meta["created_at"])}- if merged.Meta["run_id"] != "run-9" {- t.Fatalf("run_id must come from the candidate, got %v", merged.Meta["run_id"])+}++func TestWikiEntityMergeRetainsBothEvidenceBodies(t *testing.T) {+ existing := kccommon.Product{+ ID: "page-1", DocID: "kb", Variant: kccommon.VariantWiki,+ Content: "# Alpha\n\nold evidence",+ Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "page_type": "entity", "source_doc_ids": []string{"doc-1"}},+ }+ incoming := kccommon.Product{+ ID: "doc-page", DocID: "doc-2", Variant: kccommon.VariantWiki,+ Content: "# Alpha\n\nnew evidence",+ Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "page_type": "entity", "source_doc_ids": []string{"doc-2"}},+ }+ merged := wikiEntityMerge(existing, incoming)+ if merged.Content == existing.Content || merged.Content == incoming.Content {+ t.Fatalf("entity merge discarded evidence: %q", merged.Content)+ }+ if len(metaStringSliceAny(merged.Meta, "source_doc_ids")) != 2 {+ t.Fatalf("source docs = %#v, want both documents", merged.Meta["source_doc_ids"])+ }+ if merged.ID != existing.ID || merged.DocID != existing.DocID {+ t.Fatalf("entity identity changed: %#v", merged)+ }+}++func TestSplitMarkdownBlocksKeepsFencedCodeTogether(t *testing.T) {+ blocks := splitMarkdownBlocks("before\n\n```go\nline one\n\nline two\n```\n\nafter")+ if len(blocks) != 3 {+ t.Fatalf("blocks=%d, want 3: %#v", len(blocks), blocks)+ }+ if !strings.Contains(blocks[1], "line one\n\nline two") {+ t.Fatalf("fenced block was split: %q", blocks[1])}}diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 1d2721e568..ed2079fb15 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -133,6 +133,24 @@ type engineWriter struct {eng engine.DocEngine}+func (w engineWriter) DeleteMergedWikiPages(ctx context.Context, tenant, kb string, ids []string) error {+ if len(ids) == 0 {+ return nil+ }+ eng := w.eng+ if eng == nil {+ eng = engine.Get()+ }+ if eng == nil {+ return nil+ }+ baseName := fmt.Sprintf("ragflow_%s", tenant)+ _, err := eng.DeleteChunks(ctx, map[string]interface{}{+ "id": ids, "kb_id": kb, "available_int": 1, "scope_kwd": "dataset", "compile_kwd": compileKwdWikiPage,+ }, baseName, kb)+ return err+}+// datasetStructureSupported reports whether the running doc engine can filter// the dataset-structure fields (knowledge_graph_kwd + scope_kwd + raw// compile_kwd) this feature writes/deletes. Only infinity and elasticsearch@@ -646,7 +664,7 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommm["kc_kind"] = kind}// wiki_incremental port: preserve the original creation timestamp across a- // replace-only merge. If the incoming merged product already carries+ // page merge. If the incoming merged product already carries// created_at_unix (restored by the Reader from create_timestamp_flt), reuse// it; otherwise stamp a fresh now() (first creation). This is what stops every// rebuild from re-stamping the creation time.@@ -684,6 +702,12 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommif v := metaString(p.Meta, "topic"); v != "" {m["topic_kwd"] = v}+ if v := metaString(p.Meta, "plan_group"); v != "" {+ m["plan_group_kwd"] = v+ }+ if v := metaString(p.Meta, "generation"); v != "" {+ m["generation_kwd"] = v+ }if v := metaString(p.Meta, "summary"); v != "" {m["summary_with_weight"] = v}@@ -930,7 +954,7 @@ func metaInt(m map[string]any, key string) (int64, bool) {// metaFloat extracts a float64 from a map value that may be boxed as float64,// int64, int, or string — the engine/JSON round-trip does not guarantee a// single numeric type. Used to recover create_timestamp_flt so the reader can-// preserve the original creation time across a replace-only merge.+// preserve the original creation time across a page merge.func metaFloat(m map[string]any, key string) (float64, bool) {switch v := m[key].(type) {case float64:diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex a1023e812b..1c2f66f255 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -137,9 +137,10 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {}return kc.Deps{- Chat: &kcChatInvoker{svc: svc, tenantID: tenantID, llmID: llmID},- Embed: &kcEmbedder{svc: svc, tenantID: tenantID, embdID: embeddingModel},- WikiPages: &kcWikiPageStore{docEngine: engine.Get()},+ Chat: &kcChatInvoker{svc: svc, tenantID: tenantID, llmID: llmID},+ Embed: &kcEmbedder{svc: svc, tenantID: tenantID, embdID: embeddingModel},+ WikiPages: &kcWikiPageStore{docEngine: engine.Get()},+ WikiMapVersions: knowledge_compile.NewWikiMapVersionStore(engine.Get()),// HistoricalKNN / Redis are optional (wiki historical dedup,// datasetnav lock). They are wired separately when the// surrounding pipeline supplies the backing services.@@ -358,7 +359,7 @@ func (s *kcWikiPageStore) FindSimilarPages(ctx context.Context, tenantID, dataseIndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: k,- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "kc_content_md_raw", "_score"},+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "kc_content_md_raw", "_score"},// compile_kwd="wiki_page" is the schema-backed discriminator for wiki// pages (sections carry compile_kwd="wiki_section"); there is no// "kc_kind" column in the chunk schema, so filtering on it would return@@ -394,7 +395,7 @@ func (s *kcWikiPageStore) GetPageBySlug(ctx context.Context, tenantID, datasetIDIndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: []string{datasetID},Limit: 1,- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "kc_content_md_raw", "_score"},+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "kc_content_md_raw", "_score"},Filter: map[string]interface{}{"compile_kwd": "wiki_page","slug_kwd": slug,@@ -408,6 +409,35 @@ func (s *kcWikiPageStore) GetPageBySlug(ctx context.Context, tenantID, datasetIDreturn &page, nil}+func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID, datasetID string, chunkIDs []string, k int) ([]kc.WikiPageCandidate, error) {+ if s == nil || s.docEngine == nil || len(chunkIDs) == 0 || k <= 0 || strings.TrimSpace(datasetID) == "" {+ return nil, nil+ }+ req := &enginetypes.SearchRequest{+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},+ KbIDs: []string{datasetID},+ Limit: k,+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "kc_content_md_raw", "_score"},+ Filter: map[string]interface{}{+ "compile_kwd": "wiki_page",+ "source_chunk_ids": chunkIDs,+ },+ }+ res, err := s.docEngine.Search(ctx, req)+ if err != nil || res == nil {+ return nil, err+ }+ out := make([]kc.WikiPageCandidate, 0, len(res.Chunks))+ for _, row := range res.Chunks {+ candidate := wikiPageCandidateFromRow(row)+ if candidate.Score == 0 {+ candidate.Score = 0.68+ }+ out = append(out, candidate)+ }+ return out, nil+}+func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {return kc.WikiPageCandidate{ID: strings.TrimSpace(anyString(row["id"])),@@ -415,12 +445,14 @@ func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {Title: strings.TrimSpace(anyString(row["title_kwd"])),PageType: strings.TrimSpace(anyString(row["page_type_kwd"])),Topic: strings.TrimSpace(anyString(row["topic_kwd"])),+ PlanGroup: strings.TrimSpace(anyString(row["plan_group_kwd"])),Summary: strings.TrimSpace(anyString(row["summary_with_weight"])),ContentMD: strings.TrimSpace(anyString(row["content_with_weight"])),ContentMDRaw: strings.TrimSpace(anyString(row["kc_content_md_raw"])),EntityNames: anyStrings(row["entity_names_kwd"]),RelatedKBPages: anyStrings(row["related_kb_pages_kwd"]),Outlinks: anyStrings(row["outlinks_kwd"]),+ SourceChunkIDs: anyStrings(row["source_chunk_ids"]),Score: anyFloat(row["_score"]),}}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex a8bbe30d55..d63fba4611 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -212,7 +212,7 @@ func (s *PipelineExecutor) collectDebugOutput(ctx context.Context, pipelineOutpuDocID: s.taskCtx.Doc.ID,KbID: s.taskCtx.Doc.KbID,Chunks: chunks,- ChunkCount: countDistinctChunkIDs(chunks),+ ChunkCount: countOriginalChunkIDs(chunks),TokenConsumption: indexdoc.GetEmbeddingTokenConsumption(pipelineOutput),Duration: time.Since(start).Seconds(),}, nil@@ -281,7 +281,9 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput mapcommon.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))}- chunkCount := countDistinctChunkIDs(chunks)+ // Compilation products are derived artifacts and must not inflate the+ // document's source chunk counter shown by the document list API.+ chunkCount := countOriginalChunkIDs(chunks)builtInMetadata, autoMetaEnabled := builtInMetadataFromParserConfig(s.taskCtx.Doc.ParserConfig,@@ -361,15 +363,20 @@ func docNameValue(name *string) string {return *name}-// countDistinctChunkIDs returns the number of distinct chunk IDs in the slice.+// countOriginalChunkIDs returns the number of distinct source chunk IDs in the+// slice. Knowledge-compiler products are also emitted as index chunks, but+// carry compile_kwd and must not be included in the document's chunk_count.// After ProcessChunksForPipeline, every chunk carries an "id" field computed// from xxhash(text+docID). Chunks with identical text share the same id and// the search engine treats them as upserts, so the effective stored chunk count// is the number of unique ids — not len(chunks). Mirrors the index-side// deduplication that happens at write time.-func countDistinctChunkIDs(chunks []map[string]any) int {+func countOriginalChunkIDs(chunks []map[string]any) int {seen := make(map[string]struct{}, len(chunks))for _, ck := range chunks {+ if _, compiled := ck["compile_kwd"]; compiled {+ continue+ }id, _ := ck["id"].(string)if id == "" {continuediff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex e5ccd059ff..80222d12f6 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -526,9 +526,9 @@ func TestPipelineExecutorRunPipelineWithDSLForwardsSink(t *testing.T) {}}-func TestCountDistinctChunkIDs(t *testing.T) {+func TestCountOriginalChunkIDs(t *testing.T) {// Empty list- if n := countDistinctChunkIDs(nil); n != 0 {+ if n := countOriginalChunkIDs(nil); n != 0 {t.Fatalf("nil chunks: got %d, want 0", n)}@@ -538,7 +538,7 @@ func TestCountDistinctChunkIDs(t *testing.T) {{"id": "b"},{"id": "c"},}- if n := countDistinctChunkIDs(chunks); n != 3 {+ if n := countOriginalChunkIDs(chunks); n != 3 {t.Fatalf("all unique: got %d, want 3", n)}@@ -550,7 +550,7 @@ func TestCountDistinctChunkIDs(t *testing.T) {{"id": "z"},{"id": "y"}, // duplicate of [1]}- if n := countDistinctChunkIDs(chunks); n != 3 {+ if n := countOriginalChunkIDs(chunks); n != 3 {t.Fatalf("with duplicates: got %d, want 3", n)}@@ -560,7 +560,18 @@ func TestCountDistinctChunkIDs(t *testing.T) {{"text": "no id"},{"id": "two"},}- if n := countDistinctChunkIDs(chunks); n != 2 {+ if n := countOriginalChunkIDs(chunks); n != 2 {t.Fatalf("mixed present/absent ids: got %d, want 2", n)}++ // Compiler products are stored as chunks too, but are derived artifacts.+ chunks = []map[string]any{+ {"id": "source-1"},+ {"id": "source-2"},+ {"id": "wiki-page-1", "compile_kwd": "wiki_page"},+ {"id": "wiki-section-1", "compile_kwd": "wiki_section"},+ }+ if n := countOriginalChunkIDs(chunks); n != 2 {+ t.Fatalf("compiler products: got %d, want 2", n)+ }}