RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat: wiki incremental Mode A/B with durable rewrite barrier (#18122)
Zhichang Yud9ed14ce9c31
diff --git a/agent/templates/compiler.json b/agent/templates/compiler.jsonindex 3895f903eb..ce0324648c 100644--- a/agent/templates/compiler.json+++ b/agent/templates/compiler.json@@ -7,6 +7,7 @@"params": {"compilation_template_group_id": "","llm_id": "",+ "plan": false,"outputs": {"chunks": {"type": "Array<Object>",@@ -411,6 +412,7 @@"form": {"compilation_template_group_id": "","llm_id": "",+ "plan": false,"outputs": {"chunks": {"type": "Array<Object>",diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.goindex 04d68a4849..31445d54e6 100644--- a/internal/ingestion/component/knowledge_compiler/common/types.go+++ b/internal/ingestion/component/knowledge_compiler/common/types.go@@ -49,6 +49,11 @@ type Param struct {SimilarityThreshold float64MaxWorkers intEnableHistoricalDedup bool+ // Plan selects the wiki compilation mode. nil means the DSL did not specify a+ // mode, so template config may decide; the effective default is false (Mode A).+ // true selects B-mode (LLM planning + reconcile), while false selects A-mode+ // (one entity/concept = one deterministic flat page).+ Plan *bool// Extra carries arbitrary caller-provided overrides merged into the// resolved template config.Extra map[string]any@@ -168,12 +173,23 @@ func ParseParam(m map[string]any) (Param, error) {if v, ok := m["enable_historical_dedup"].(bool); ok {p.EnableHistoricalDedup = v}+ // Keep presence separate from the boolean value: nil means template config may+ // supply the mode, while an explicit false still overrides template plan:true.+ if v, ok := m["plan"].(bool); ok {+ p.Plan = &v+ }if raw, ok := m["extra"].(map[string]any); ok {p.Extra = raw}return p, nil}+// PlanEnabled reports whether wiki B-mode is selected. An unset value defaults+// to Mode A.+func (p Param) PlanEnabled() bool {+ return p.Plan != nil && *p.Plan+}+// KindToVariant maps a compilation_template.kind to the Go compiler Variant.//// The Python model uses richer kind values (mind_map, page_index,diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex fa44d109cb..a52135dea8 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -440,6 +440,14 @@ func overlayTemplateConfig(param *common.Param, cfg map[string]any) {if v, ok := cfg["enable_historical_dedup"].(bool); ok {param.EnableHistoricalDedup = v}+ if param.Plan == nil {+ if v, ok := cfg["no_plan"].(bool); ok && v {+ disabled := false+ param.Plan = &disabled+ } else if v, ok := cfg["plan"].(bool); ok {+ param.Plan = &v+ }+ }// llm_id / embedding_model are optional per-call overrides documented on// Invoke. The template config supplies defaults, so only apply them when// the caller has not already provided an explicit value (the caller wins).diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex a9aaa5418a..d9f796e81a 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -730,7 +730,7 @@ func TestKnowledgeCompiler_Wiki_HistoricalDedupDropsDuplicates(t *testing.T) {installVariantTemplateResolver(t, "wiki")c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{- "compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1",+ "compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1", "plan": true,})if err != nil {t.Fatalf("NewKnowledgeCompilerComponent: %v", err)@@ -789,7 +789,7 @@ func TestKnowledgeCompiler_Wiki_UpdateMergesExistingPage(t *testing.T) {installVariantTemplateResolver(t, "wiki")c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{- "compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1",+ "compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1", "plan": true,})if err != nil {t.Fatalf("NewKnowledgeCompilerComponent: %v", err)@@ -1296,6 +1296,53 @@ func TestKnowledgeCompiler_TenantFromGlobals(t *testing.T) {}}+func TestOverlayTemplateConfigPlanPrecedence(t *testing.T) {+ boolPtr := func(value bool) *bool { return &value }++ cases := []struct {+ name string+ initial *bool+ cfg map[string]any+ want bool+ }{+ {+ name: "unset_uses_template_plan",+ cfg: map[string]any{"plan": true},+ want: true,+ },+ {+ name: "no_plan_disables_plan",+ cfg: map[string]any{"no_plan": true, "plan": true},+ want: false,+ },+ {+ name: "explicit_dsl_false_overrides_template",+ initial: boolPtr(false),+ cfg: map[string]any{"plan": true},+ want: false,+ },+ {+ name: "explicit_dsl_true_overrides_no_plan",+ initial: boolPtr(true),+ cfg: map[string]any{"no_plan": true},+ want: true,+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ param := common.Param{Plan: tc.initial}+ overlayTemplateConfig(¶m, tc.cfg)+ if param.Plan == nil {+ t.Fatal("Plan = nil after template config")+ }+ if *param.Plan != tc.want {+ t.Errorf("Plan = %t, want %t", *param.Plan, tc.want)+ }+ })+ }+}+// TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks locks the chunk-carrier// contract: the upstream chunker hands chunks over as []map[string]any (see the// chunk map shape {"id","text","ck_type","doc_type_kwd","tk_nums"} observed fromdiff --git a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go b/internal/ingestion/component/knowledge_compiler/wiki/prompt.goindex 033f745824..25ddba76c7 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/prompt.go@@ -227,6 +227,9 @@ const wikiRefineWriterUserTemplate = `## Task## Available pages (ONLY use these slugs for [[wikilinks]]){all_plan_slugs}+## Related KB pages (cross-link only those that are also in the available pages list above)+{related_kb_pages}+{existing_section}## Source document textdiff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex 1a2097ddd6..d5caebdf07 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -365,9 +365,23 @@ func (p *wikiPipeline) run() error {zap.String("doc_id", p.runKey()),zap.Int("entities", len(p.reduced.Entities)),zap.Int("claims", len(p.reduced.Claims)))- plan, err := p.runPlan()- if err != nil {- return err+ // PLAN (B-mode only): LLM-based page plan + reconcile against existing pages.+ // Mode A (param.PlanEnabled() == false, the default) skips the planner entirely — every+ // extracted entity/concept becomes its own flat page (1 identity = 1 page), so+ // the wiki is a flat encyclopedia without PLAN-grouped pages. See buildModeAPlan.+ var plan wikiPlan+ var err error+ if p.param.PlanEnabled() {+ plan, err = p.runPlan()+ if err != nil {+ return err+ }+ appcommon.Info("wiki: PLAN (B-mode) done", zap.String("dataset_id", p.datasetID), zap.String("doc_id", p.runKey()), zap.Int("plan_pages", len(plan.Pages)))+ } else {+ // wiki_incremental port (T1 + M1 Mode A): deterministic flat plan from the+ // reduced graph. No LLM, no reconcile.+ plan = p.buildModeAPlan()+ appcommon.Info("wiki: PLAN (A-mode flat) done", zap.String("dataset_id", p.datasetID), zap.String("doc_id", p.runKey()), zap.Int("plan_pages", len(plan.Pages)))}p.plan = planappcommon.Info("wiki: PLAN done",@@ -762,6 +776,7 @@ func (p *wikiPipeline) runRefinePage("title": firstNonEmpty(planItem.Title, planItem.Slug),"page_type": firstNonEmpty(planItem.PageType, "concept"),"all_plan_slugs": strings.Join(available, "\n"),+ "related_kb_pages": formatWikiRelatedKB(planItem.RelatedKB, pageTitles),"existing_section": existingSection,"source_context": sourceContext,"evidence_count": fmt.Sprintf("%d", len(evidence)),@@ -794,6 +809,14 @@ func (p *wikiPipeline) runRefinePage(}}contentRendered, outlinks := transformWikiLinks(contentRaw, firstNonEmpty(p.datasetID, p.docID), pageTitles, slugToPageType)+ // wiki_incremental port (O2): deterministically guarantee a "See also"+ // cross-link section from RelatedKB. transformWikiLinks/resolveSlug drop+ // links whose target cannot be resolved to a known slug, so an LLM-authored+ // page could silently lose every RelatedKB edge. We append the related pages+ // AFTER transformWikiLinks and do NOT re-run the resolver, so dead links are+ // bypassed and the missing target does not drop the edge — each RelatedKB+ // full-slug is linked once, only if not already present in Outlinks.+ contentRendered, outlinks = appendWikiSeeAlso(contentRendered, outlinks, planItem.RelatedKB, p.datasetID, pageTitles, slugToPageType)sourceDocIDs := collectWikiSourceDocIDs(p.inputs.Chunks, sourceChunkIDs, p.docID)summary := firstParagraph(contentRendered)if summary == "" {@@ -1545,6 +1568,81 @@ func wikiTitleKey(pageType, title string) string {return strings.TrimSpace(pageType) + "\x00" + normKey(title)}+// buildModeAPlan synthesizes a FLAT wiki plan (Mode A) directly from the reduced+// extract graph — no LLM planner, no reconcile. Every extracted entity and+// concept becomes its own canonical page (1 identity = 1 page), slugged as+// "entity/<slug>" / "concept/<slug>". Cross-links (RelatedKB) are derived purely+// from the reduced relations: each relation end is mapped to its full-slug page+// and the counterpart is added to the page's RelatedKB. This is the deterministic+// counterpart of Python's no_plan wiki mode.+func (p *wikiPipeline) buildModeAPlan() wikiPlan {+ reduced := p.reduced+ // Build the full-slug index for every entity/concept so relation endpoints+ // (which are bare names) resolve to canonical pages.+ fullSlugFor := func(name, pageType string) string {+ return pageType + "/" + normalizeWikiSlugHyphens(slugify(name))+ }+ slugToIndex := map[string]int{}+ var pages []wikiPlanPage+ addPage := func(slug, title, pageType string, entityNames []string) {+ if _, ok := slugToIndex[slug]; ok {+ return+ }+ page := wikiPlanPage{+ Action: "CREATE",+ Slug: slug,+ Title: title,+ PageType: pageType,+ Topic: title,+ EntityNames: entityNames,+ Priority: len(pages) + 1,+ }+ slugToIndex[slug] = len(pages)+ pages = append(pages, page)+ }+ for _, e := range reduced.Entities {+ name := strings.TrimSpace(e.Name)+ if name == "" {+ continue+ }+ slug := fullSlugFor(name, "entity")+ addPage(slug, name, "entity", []string{name})+ }+ for _, c := range reduced.Concepts {+ term := strings.TrimSpace(c.Term)+ if term == "" {+ continue+ }+ slug := fullSlugFor(term, "concept")+ addPage(slug, term, "concept", []string{term})+ }+ // Resolve relations into RelatedKB (full-slug cross-links) on both endpoints.+ for _, rel := range reduced.Relations {+ from := strings.TrimSpace(rel.From)+ to := strings.TrimSpace(rel.To)+ if from == "" || to == "" {+ continue+ }+ fromSlug := fullSlugFor(from, "entity")+ toSlug := fullSlugFor(to, "entity")+ if fromSlug == toSlug {+ continue+ }+ if i, ok := slugToIndex[fromSlug]; ok {+ pages[i].RelatedKB = append(pages[i].RelatedKB, toSlug)+ }+ if i, ok := slugToIndex[toSlug]; ok {+ pages[i].RelatedKB = append(pages[i].RelatedKB, fromSlug)+ }+ }+ for i := range pages {+ pages[i].RelatedKB = uniqueStrings(pages[i].RelatedKB)+ }+ plan := wikiPlan{Pages: pages}+ plan.Pages = normalizeWikiPlanPages(plan.Pages, reduced)+ return plan+}+func buildWikiFallbackPages(reduced wikiExtract) []wikiPlanPage {var out []wikiPlanPageseen := map[string]bool{}@@ -2159,47 +2257,64 @@ func pageTypeOf(slug string, slugToPageType map[string]string) string {return "page"}-func transformWikiLinks(content, kbID string, pageTitles, slugToPageType map[string]string) (string, []string) {- kbID = strings.TrimSpace(kbID)- // Resolve a wikitext slug (which may be a bare "name" or a full- // "<page_type>/<slug>") to the canonical full slug used as the page- // identifier (slug_kwd). This mirrors Python's _wiki_resolve_dead_slug:- // plain names / titles are reverse-mapped to the full pid, and slugs that- // cannot be resolved are dropped (dead links). Without this, bare slugs- // written by the LLM never match the full-slug page index, so wiki- // relations (edges) are silently lost.- bareToSlug := map[string]string{}- titleToSlug := map[string]string{}- // Plan slugs are canonicalized to the hyphen style upstream- // (normalizeWikiPlanPage -> normalizeWikiSlugHyphens), so slugToPageType /- // pageTitles keys are already hyphen full-slugs. LLM-authored wikitext- // links may still carry underscores (e.g. "dong_zhuo"); normalize both- // sides to hyphens so the reverse lookup matches, and always emit the- // resolved outlink in the canonical hyphen full-slug form.+// wikiSlugResolver resolves a wikitext slug (a bare "name" or a full+// "<page_type>/<slug>") to the canonical hyphen full-slug used as the page+// identifier (slug_kwd), mirroring Python's _wiki_resolve_dead_slug: plain+// names / titles are reverse-mapped to the full pid. An empty result means the+// target is not a known page and must not be emitted as a link or graph edge.+type wikiSlugResolver struct {+ slugToPageType map[string]string+ bareToSlug map[string]string+ titleToSlug map[string]string+}++// newWikiSlugResolver builds the reverse lookup maps from the available-page+// index. Plan slugs are canonicalized to the hyphen style upstream+// (normalizeWikiPlanPage -> normalizeWikiSlugHyphens), so slugToPageType /+// pageTitles keys are already hyphen full-slugs. LLM-authored wikitext links may+// still carry underscores (e.g. "dong_zhuo"); normalize both sides to hyphens so+// the reverse lookup matches.+func newWikiSlugResolver(pageTitles, slugToPageType map[string]string) *wikiSlugResolver {+ r := &wikiSlugResolver{+ slugToPageType: slugToPageType,+ bareToSlug: map[string]string{},+ titleToSlug: map[string]string{},+ }for fullSlug := range slugToPageType {- bareToSlug[normalizeWikiSlugHyphens(lastPathSlug(fullSlug))] = fullSlug+ r.bareToSlug[normalizeWikiSlugHyphens(lastPathSlug(fullSlug))] = fullSlug}for fullSlug, title := range pageTitles {if t := strings.TrimSpace(title); t != "" {- titleToSlug[t] = fullSlug+ r.titleToSlug[t] = fullSlug}}- resolveSlug := func(slug string) string {- slug = strings.TrimSpace(slug)- if slug == "" {- return ""- }- if _, ok := slugToPageType[slug]; ok {- return slug- }- if full, ok := bareToSlug[normalizeWikiSlugHyphens(lastPathSlug(slug))]; ok {- return full- }- if full, ok := titleToSlug[slug]; ok {- return full- }+ return r+}++// resolve returns the canonical hyphen full-slug for target, or "" when the+// target is not a known page (dead link). The result is always emitted in the+// canonical hyphen form so outlinks_kwd and slug_kwd agree in format.+func (r *wikiSlugResolver) resolve(target string) string {+ slug := strings.TrimSpace(target)+ if slug == "" {return ""}+ if _, ok := r.slugToPageType[slug]; ok {+ return normalizeWikiSlugHyphens(slug)+ }+ if full, ok := r.bareToSlug[normalizeWikiSlugHyphens(lastPathSlug(slug))]; ok {+ return normalizeWikiSlugHyphens(full)+ }+ if full, ok := r.titleToSlug[slug]; ok {+ return normalizeWikiSlugHyphens(full)+ }+ return ""+}++func transformWikiLinks(content, kbID string, pageTitles, slugToPageType map[string]string) (string, []string) {+ kbID = strings.TrimSpace(kbID)+ resolver := newWikiSlugResolver(pageTitles, slugToPageType)+ resolveSlug := resolver.resolveseen := map[string]bool{}var outlinks []stringtrack := func(slug string) {@@ -2317,6 +2432,91 @@ func transformWikiLinks(content, kbID string, pageTitles, slugToPageType map[strreturn out, outlinks}+// formatWikiRelatedKB renders the RelatedKB page list for the refine-writer+// prompt (O1): one "- [[<slug>]]" bullet per related full-slug, with the page+// title as display text when known. It is a prompt hint only — the authoritative+// See-also section is appended deterministically by appendWikiSeeAlso after the+// LLM output is transformed.+func formatWikiRelatedKB(related []string, pageTitles map[string]string) string {+ if len(related) == 0 {+ return "(none)"+ }+ var b strings.Builder+ for _, slug := range related {+ slug = strings.TrimSpace(slug)+ if slug == "" {+ continue+ }+ canon := normalizeWikiSlugHyphens(slug)+ if title := strings.TrimSpace(pageTitles[canon]); title != "" {+ b.WriteString("- [[" + canon + "|" + title + "]]\n")+ } else {+ b.WriteString("- [[" + canon + "]]\n")+ }+ }+ if b.Len() == 0 {+ return "(none)"+ }+ return strings.TrimRight(b.String(), "\n")+}++// appendWikiSeeAlso guarantees a cross-link section from RelatedKB (O2). It runs+// AFTER transformWikiLinks, which drops dead links (targets that cannot be+// resolved). To avoid losing every RelatedKB edge when a target is missing, we do+// NOT re-run the resolver here: each RelatedKB full-slug is linked once as+// artifact/<kb>/<page_type>/<slug>, only if it is not already present in+// outlinks. This bypasses the dead-link drop and keeps the graph edge.+func appendWikiSeeAlso(content string, outlinks, related []string, kbID string, pageTitles, slugToPageType map[string]string) (string, []string) {+ if len(related) == 0 {+ return content, outlinks+ }+ resolver := newWikiSlugResolver(pageTitles, slugToPageType)+ have := make(map[string]bool, len(outlinks))+ for _, o := range outlinks {+ have[normalizeWikiSlugHyphens(o)] = true+ }+ var bullets []string+ for _, slug := range related {+ // Resolve the RelatedKB target against the current page index. An+ // unresolved target is not a known page: omit it entirely (no broken+ // artifact link) and do NOT append it to outlinks, so no phantom graph+ // edge is persisted for a page that was never compiled.+ canon := resolver.resolve(slug)+ if canon == "" || have[canon] {+ continue+ }+ have[canon] = true+ bareSlug := canon+ pageType := pageTypeOf(canon, slugToPageType)+ if idx := strings.Index(canon, "/"); idx >= 0 && canon[:idx] == pageType {+ bareSlug = canon[idx+1:]+ }+ if pageType == "" {+ pageType = "page"+ }+ label := pageTitles[canon]+ if label == "" {+ label = strings.Title(strings.ReplaceAll(lastPathSlug(canon), "-", " "))+ }+ bullets = append(bullets, "- ["+label+"](artifact/"+kbID+"/"+pageType+"/"+bareSlug+")")+ outlinks = append(outlinks, canon)+ }+ if len(bullets) == 0 {+ return content, outlinks+ }+ // Append a "See also" section if one is not already present; otherwise add+ // the bullets under the existing "See also" heading.+ body := strings.TrimRight(content, "\n")+ if seeAlsoRe.MatchString(body) {+ return body + "\n" + strings.Join(bullets, "\n") + "\n", outlinks+ }+ return body + "\n\n## See also\n\n" + strings.Join(bullets, "\n") + "\n", outlinks+}++// seeAlsoRe matches a Markdown "See also" heading (case-insensitive) so we can+// append related links under an existing section instead of duplicating it.+var seeAlsoRe = regexp.MustCompile(`(?m)^#{1,6}\s*see\s*also\s*$`)+func firstN(s []string, n int) []string {if len(s) <= n {return sdiff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex 8f7a61b38f..753b017201 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -17,18 +17,26 @@ package knowledge_compileimport ("context"+ "errors""fmt""sort""sync""time""ragflow/internal/common"+ "ragflow/internal/dao""ragflow/internal/engine"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common""go.uber.org/zap"+ "gorm.io/gorm")+// kcDB is the package-level MySQL handle installed by Provision. It backs the+// doc enumeration used by RebuildDataset; it is nil (and docLister returns no+// docs) when the scheduler was provisioned without a DB.+var kcDB *gorm.DB+// Consumer is the dataset-level post-processing worker (§11.5). Multiple// instances compete on the MySQL scheduling rows; each KB is processed by at// most one instance at a time via the per-KB claim (so the same KB is handled@@ -49,6 +57,20 @@ type Consumer struct {mu sync.Mutextombs map[string]map[string]uint64 // dataset -> docID -> delete marker (tombstone)++ // rebuildPause suppresses local claims while this consumer rebuilds a dataset.+ rebuildPause map[string]bool++ // docLister enumerates every doc id in a KB so a rewrite can republish them.+ docLister func(ctx context.Context, tenant, kb string) ([]string, error)+}++// rewriteScheduler is the subset of the scheduler API a dataset rewrite needs,+// satisfied by *mysqlScheduler (and FakeScheduler in tests). Using a narrow+// interface keeps the Claimer surface unchanged.+type rewriteScheduler interface {+ Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error+ CancelInflight(ctx context.Context, datasetID, token string) error}// NewConsumer constructs a Consumer driven by the given Claimer. Tests pass a@@ -65,6 +87,8 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {sweepInterval: 30 * time.Second,mergeThreshold: 0.85,tombs: map[string]map[string]uint64{},+ rebuildPause: map[string]bool{},+ docLister: defaultDocLister,}for _, o := range opts {o(c)@@ -135,6 +159,13 @@ func (c *Consumer) tryClaimAndProcess(ctx context.Context) {func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {datasetID := cr.DatasetID+ c.mu.Lock()+ paused := c.rebuildPause[datasetID]+ c.mu.Unlock()+ if paused {+ return+ }+// Heartbeat refreshes the claim TTL while we process; a failed touch means// the lease was taken over (or reclaimed) and we must abort without acking.stopHb := make(chan struct{})@@ -165,7 +196,7 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {var batchErr errorgo func() {defer close(done)- batchErr = c.processBatch(ctx, cr.TenantID, datasetID, cr.Entries)+ batchErr = c.processBatch(ctx, cr.TenantID, datasetID, cr.Token, cr.Entries)}()select {@@ -185,6 +216,9 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {// must leave the claimed batch in the backlog for reclamation/retry rather// than silently dropping it (C5: never ack what we failed to merge).if batchErr != nil {+ if errors.Is(batchErr, errClaimSuperseded) {+ return+ }common.Error("knowledge_compile: batch processing failed, leaving batch for retry",batchErr,zap.String("dataset_id", datasetID),@@ -201,11 +235,142 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {}}+var errClaimSuperseded = errors.New("knowledge_compile: claim superseded by rewrite")++// withWriteLock runs a destructive side effect fn under the scheduler's+// per-dataset write/rebuild lock, verifying the claim token inside the lock+// immediately before fn. This closes the TOCTOU gap between cancellation and a+// writer side effect: a cancelled worker cannot write after the rebuild clears+// storage, and a rebuild cannot interleave while fn runs.+func (c *Consumer) withWriteLock(ctx context.Context, kb, token string, fn func() error) error {+ return c.scheduler.WithDatasetLock(ctx, kb, func(currentToken string) error {+ if currentToken != token {+ return errClaimSuperseded+ }+ return fn()+ })+}++// RebuildDataset performs a full incremental rewrite of a KB's dataset-level+// merged products (W1/W2/W5/W6). The order is fixed to close both the+// enumerate-then-clear race (M19/C-race) AND the cross-process stale-write+// window:+// 1. set the rewrite pause (in-process only; clears via defer on every path);+// 2. CancelInflight invalidates any in-flight claim before clearing storage.+// Each worker compares its claim token under the write lock and therefore+// cannot write after cancellation;+// 3. DeleteMerged + DropWikiGraph clear the old merged + graph state INSIDE the+// same per-dataset row lock (WithDatasetLock). This is what actually closes+// the TOCTOU gap: the generation check + write of every worker run under this+// lock, so the rebuild's clear either runs after any in-flight worker write+// has finished (it is later removed by the clear) or after a stale worker has+// self-dropped — a worker can never repopulate cleared state with old results+// because it cannot hold the lock concurrently with the clear;+// 4. enumerate every doc and republish;+// 5. clear the local pause.+//+// mode is "incremental" or "rewrite"; today both take the same clean-and-rebuild+// path, with mode retained for future differential strategies.+func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string) error {+ rs, ok := c.scheduler.(rewriteScheduler)+ if !ok {+ return fmt.Errorf("knowledge_compile: scheduler %T does not support rewrite", c.scheduler)+ }++ // 1. pause the rewrite window. The pause is cleared by a deferred cleanup so a+ // failure mid-rebuild (DeleteMerged/DropWikiGraph/CancelInflight/Bump/Publish)+ // can never leave the dataset permanently paused (which would drop every+ // future claim). Partially published docs are retried on the next claim.+ c.mu.Lock()+ c.rebuildPause[kb] = true+ c.mu.Unlock()+ defer func() {+ c.mu.Lock()+ c.rebuildPause[kb] = false+ c.mu.Unlock()+ }()++ // 2. Cancel any in-flight claim before clearing storage. A worker that was+ // already running observes its revoked claim token inside withWriteLock and+ // returns without writing.+ if err := rs.CancelInflight(ctx, kb, ""); err != nil {+ return fmt.Errorf("knowledge_compile: rebuild cancel inflight: %w", err)+ }++ // 3. clear old merged + graph state (structural filter is the source of+ // truth; no per-doc rows are touched) under the per-dataset row lock. This+ // guarantees mutual exclusion with every worker's destructive write (which+ // runs under the same lock via withWriteLock): the clear either waits for an+ // in-flight worker write to finish (then removes its old-generation result)+ // or runs after stale workers have self-dropped. A worker cannot hold the+ // lock while the clear runs, so it can never repopulate cleared state with+ // old results.+ if err := c.scheduler.WithDatasetLock(ctx, kb, func(_ string) error {+ if derr := c.writer.DeleteMerged(ctx, tenant, kb); derr != nil {+ return derr+ }+ return c.writer.DropWikiGraph(ctx, tenant, kb)+ }); err != nil {+ return fmt.Errorf("knowledge_compile: rebuild clear merged+graph: %w", err)+ }++ // 4. enumerate every doc and republish.+ docs, err := c.docLister(ctx, tenant, kb)+ if err != nil {+ return fmt.Errorf("knowledge_compile: rebuild list docs: %w", err)+ }+ for _, docID := range docs {+ if perr := rs.Publish(ctx, tenant, kb, docID, string(EventTypeCompleted)); perr != nil {+ return fmt.Errorf("knowledge_compile: rebuild republish %s: %w", docID, perr)+ }+ }++ // 6. resweep: the deferred cleanup clears the pause so any claim arriving+ // post-enumeration is processed (and stale ones self-drop in processClaim).+ common.Info("knowledge_compile: dataset rebuild complete",+ zap.String("dataset_id", kb),+ zap.String("mode", mode),+ zap.Int("docs", len(docs)))+ return nil+}++// defaultDocLister enumerates every doc id in a KB via the Document DAO. When+// no DB was provisioned it returns no docs (a rewrite becomes a no-op clean).+func defaultDocLister(ctx context.Context, tenant, kb string) ([]string, error) {+ if kcDB == nil {+ return nil, nil+ }+ return dao.NewDocumentDAO().ListIDsByKBIDWithOptions(ctx, kcDB, dao.DocumentListOptions{KbID: kb})+}++// filterWikiPageCandidates returns only the candidates the dataset-level merge+// is allowed to fold: for the wiki variant, that is strictly page-kind products+// (Meta.kind == "page"). Sections are a doc-level concern and must never enter+// the dataset-level page bucket, and the deduper must not carry an implicit+// section-filter contract. Non-wiki variants pass through unchanged. Legacy wiki+// rows whose kind is empty are derived in the Reader; only truly page-kind wiki+// products proceed.+func filterWikiPageCandidates(candidates []kccommon.Product) []kccommon.Product {+ // Allocate a fresh slice: reusing candidates[:0] would overwrite the caller's+ // backing array in place, corrupting any other reference (e.g. the vector+ // audit that still reads `incoming`).+ filtered := make([]kccommon.Product, 0, len(candidates))+ for _, cand := range candidates {+ if cand.Variant == kccommon.VariantWiki {+ if metaString(cand.Meta, "kind") != "page" {+ continue+ }+ }+ filtered = append(filtered, cand)+ }+ return filtered+}+// processBatch applies out-of-order / tombstone handling, then recomputes and// writes the dataset-level merged products for the claimed closed batch. It// returns an error if any reader/dedup/writer step fails so the caller can// leave the batch for reclamation instead of acking dropped work.-func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries []BacklogEntry) error {+func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, entries []BacklogEntry) error {common.Info("knowledge_compile: processing claimed batch",zap.String("dataset_id", kb),zap.String("tenant_id", tenant),@@ -326,10 +491,31 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entriesfor d := range deletedSet {delIDs = append(delIDs, d)}- if err := c.writer.DeleteDocLevelForDocs(ctx, tenant, kb, delIDs); err != nil {+ // Each destructive write runs under the scheduler's per-dataset+ // write/rebuild lock with the generation check performed INSIDE the lock,+ // so a rewrite can neither land between the check and the write nor+ // interleave with the write itself (it must wait for this lock). This+ // closes the TOCTOU window where a worker past its fence could repopulate+ // storage the rebuild just cleared.+ if err := c.withWriteLock(ctx, kb, token, func() error {+ return c.writer.DeleteDocLevelForDocs(ctx, tenant, kb, delIDs)+ }); err != nil {+ if errors.Is(err, errClaimSuperseded) {+ common.Info("knowledge_compile: batch stale before delete, aborting (rewrite barrier)",+ zap.String("dataset_id", kb))+ }return err}- if err := c.writer.StripMergedSources(ctx, tenant, kb, delIDs); err != nil {+ // The second destructive call is its own locked section: a rewrite can+ // land between the two, in which case the stale batch must not apply its+ // second side effect under the new generation.+ if err := c.withWriteLock(ctx, kb, token, func() error {+ return c.writer.StripMergedSources(ctx, tenant, kb, delIDs)+ }); err != nil {+ if errors.Is(err, errClaimSuperseded) {+ common.Info("knowledge_compile: batch stale before strip, aborting (rewrite barrier)",+ zap.String("dataset_id", kb))+ }return err}}@@ -351,10 +537,19 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entriesincoming = append(incoming, docProducts...)}+ // wiki_incremental port (M1): the dataset-level merge only processes wiki+ // PAGES. A wiki doc yields both page and section products (Meta.kind+ // "page"/"section"); sections are a doc-level concern and must never be folded+ // into the dataset-level page bucket. Filter BEFORE the in-memory dedup so+ // sections never enter the deduper (and never trigger LLM/embedding/alias+ // processing) — the decider must not carry an implicit section-filter+ // contract. Legacy rows whose kind is empty are derived in the Reader; only+ // truly page-kind wiki products proceed.+ candidates := filterWikiPageCandidates(incoming)// In-memory dedup among the completed batch first.- candidates, err := deduper.Dedup(ctx, incoming)- if err != nil {- return err+ candidates, dedupErr := deduper.Dedup(ctx, candidates)+ if dedupErr != nil {+ return dedupErr}// Diagnostics: after batch dedup, verify the per-doc products still carry// their embedding before the KNN/merge path consumes cand.Vector. If@@ -433,7 +628,12 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entriesvec64[i] = float64(v)}}- hit, score, err := c.reader.SearchSimilar(ctx, tenant, kb, cand.Variant, vec64, 1, c.mergeThreshold)+ // wiki_incremental port (B2): request topN >= 2 so SearchSimilar can+ // apply its score-descending skip rule on a dirty top-1 (the reader+ // drops rows whose compile_kwd does not map to the searched variant;+ // a re-query with topN=2 lets the next clean candidate surface instead+ // of falling through to "no hit").+ hit, score, err := c.reader.SearchSimilar(ctx, tenant, kb, cand.Variant, vec64, 2, c.mergeThreshold)if err != nil {return err}@@ -518,11 +718,21 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries}}- // Write the surviving merged set (updated existing + new distinct rows).+ // Write the surviving merged set (updated existing + new distinct rows). The+ // generation check + WriteMerged run atomically under the per-dataset+ // write/rebuild lock, so a rewrite that lands during the (now long) KNN + LLM+ // merge cannot leak into the freshly rewritten index, and a rewrite that is+ // clearing storage cannot interleave with the write.mergedFinal := make([]kccommon.Product, 0, len(newMerged)+len(unmatched))mergedFinal = append(mergedFinal, newMerged...)mergedFinal = append(mergedFinal, unmatched...)- if err := c.writer.WriteMerged(ctx, tenant, kb, mergedFinal); err != nil {+ if err := c.withWriteLock(ctx, kb, token, func() error {+ return c.writer.WriteMerged(ctx, tenant, kb, mergedFinal)+ }); err != nil {+ if errors.Is(err, errClaimSuperseded) {+ common.Info("knowledge_compile: batch stale before write, aborting (rewrite barrier)",+ zap.String("dataset_id", kb))+ }return err}@@ -532,8 +742,15 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries// when the current batch only prunes (deletes) wiki pages, and even for a// non-wiki batch where the dataset already has no wiki pages (ProjectWikiGraph// then just drops any stale graph rows). The graph is reconstructible, so the- // cost of an occasional no-op reprojection is accepted.- if err := c.writer.ProjectWikiGraph(ctx, tenant, kb); err != nil {+ // cost of an occasional no-op reprojection is accepted. It is also a locked+ // destructive side effect for the same TOCTOU reasons as WriteMerged.+ if err := c.withWriteLock(ctx, kb, token, func() error {+ return c.writer.ProjectWikiGraph(ctx, tenant, kb)+ }); err != nil {+ if errors.Is(err, errClaimSuperseded) {+ common.Info("knowledge_compile: batch stale before graph projection, aborting (rewrite barrier)",+ zap.String("dataset_id", kb))+ }return err}diff --git a/internal/ingestion/knowledge_compile/consumer_test.go b/internal/ingestion/knowledge_compile/consumer_test.godeleted file mode 100644index 5104ad0dae..0000000000--- a/internal/ingestion/knowledge_compile/consumer_test.go+++ /dev/null@@ -1,424 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.--package knowledge_compile--import (- "context"- "sync"- "testing"- "time"-- kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"-)--// fakeReader returns a fixed per-document product set, keyed by docID.-type fakeReader struct {- mu sync.Mutex- products []kccommon.Product- calls int-}--func (r *fakeReader) LoadDocProducts(_ context.Context, _, _, docID string) ([]kccommon.Product, error) {- r.mu.Lock()- defer r.mu.Unlock()- r.calls++- var out []kccommon.Product- for _, p := range r.products {- if p.DocID == docID {- out = append(out, p)- }- }- return out, nil-}--func (r *fakeReader) SearchSimilar(_ context.Context, _, _ string, _ kccommon.Variant, _ []float64, _ int, _ float64) (kccommon.Product, float64, error) {- return kccommon.Product{}, 0, nil-}--// fakeWriter captures written merged products.-type fakeWriter struct {- mu sync.Mutex- written [][]kccommon.Product- deletedDocLevel []string- strippedSources []string- projected []string- dropped []string-}--func (w *fakeWriter) WriteMerged(_ context.Context, _, _ string, products []kccommon.Product) error {- if len(products) == 0 {- return nil- }- w.mu.Lock()- defer w.mu.Unlock()- cp := make([]kccommon.Product, len(products))- copy(cp, products)- w.written = append(w.written, cp)- return nil-}--func (w *fakeWriter) DeleteDocLevelForDocs(_ context.Context, _, _ string, docIDs []string) error {- w.mu.Lock()- defer w.mu.Unlock()- w.deletedDocLevel = append(w.deletedDocLevel, docIDs...)- return nil-}--func (w *fakeWriter) StripMergedSources(_ context.Context, _, _ string, docIDs []string) error {- w.mu.Lock()- defer w.mu.Unlock()- w.strippedSources = append(w.strippedSources, docIDs...)- return nil-}--func (w *fakeWriter) ProjectWikiGraph(_ context.Context, _, kb string) error {- w.mu.Lock()- defer w.mu.Unlock()- w.projected = append(w.projected, kb)- return nil-}--func (w *fakeWriter) DropWikiGraph(_ context.Context, _, kb string) error {- w.mu.Lock()- defer w.mu.Unlock()- w.dropped = append(w.dropped, kb)- return nil-}--func sampleProducts() []kccommon.Product {- return []kccommon.Product{- {ID: "p1", DocID: "d1", TenantID: "t1", Variant: kccommon.Variant("structure"),- Content: `{"name":"X"}`, Meta: map[string]any{"name": "X", "kind": "entity"}},- {ID: "p2", DocID: "d1", TenantID: "t1", Variant: kccommon.Variant("structure"),- Content: `{"name":"Y"}`, Meta: map[string]any{"name": "Y", "kind": "entity"}},- }-}--func newTestConsumer(sch *FakeScheduler, r *fakeReader, w *fakeWriter, factory DeduperFactory) *Consumer {- return NewConsumer(sch,- WithReader(r),- WithWriter(w),- WithDeduperFactory(factory),- )-}--func TestConsumerCompletedWritesMerged(t *testing.T) {- sch := NewFakeScheduler()- r := &fakeReader{products: sampleProducts()}- w := &fakeWriter{}- c := newTestConsumer(sch, r, w, func(string) (Deduper, error) { return NewNoopDeduper(), nil })-- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {- t.Fatalf("append: %v", err)- }- c.tryClaimAndProcess(context.Background())-- w.mu.Lock()- defer w.mu.Unlock()- if len(w.written) != 1 {- t.Fatalf("expected 1 WriteMerged call, got %d", len(w.written))- }- if len(w.written[0]) != 2 {- t.Fatalf("expected 2 merged products, got %d", len(w.written[0]))- }- if len(w.projected) != 1 || w.projected[0] != "kb1" {- t.Fatalf("expected 1 ProjectWikiGraph(kb1) call, got %v", w.projected)- }- // After ack, the claim row must be cleared (no live lease left behind).- if _, ok, _ := sch.TryClaim(context.Background()); ok {- t.Fatalf("expected no claimable row after ack")- }-}--func TestConsumerTombstoneSkipsCompletedBeforeDeleted(t *testing.T) {- sch := NewFakeScheduler()- r := &fakeReader{products: sampleProducts()}- w := &fakeWriter{}- c := newTestConsumer(sch, r, w, func(string) (Deduper, error) { return NewNoopDeduper(), nil })-- // completed before deleted: the deletion is the last event for d1, so it- // wins and d1's per-doc products are orphaned (completion skipped).- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {- t.Fatalf("append completed: %v", err)- }- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeDeleted)); err != nil {- t.Fatalf("append deleted: %v", err)- }- c.tryClaimAndProcess(context.Background())-- w.mu.Lock()- defer w.mu.Unlock()- // No merged write (completed skipped, since deletion is the last event).- // The deletion is handled entirely on the DocEngine: d1's per-doc products- // are dropped in one call and d1 is stripped from every dataset-level- // product in one call. No products are loaded into memory.- if len(w.written) != 0 {- t.Fatalf("expected no merged write, got %d", len(w.written))- }- if len(w.deletedDocLevel) != 1 || w.deletedDocLevel[0] != "d1" {- t.Fatalf("expected DeleteDocLevelForDocs([d1]), got %v", w.deletedDocLevel)- }- if len(w.strippedSources) != 1 || w.strippedSources[0] != "d1" {- t.Fatalf("expected StripMergedSources([d1]), got %v", w.strippedSources)- }-}--func TestConsumerReingestAfterDeletionWins(t *testing.T) {- sch := NewFakeScheduler()- r := &fakeReader{products: sampleProducts()}- w := &fakeWriter{}- c := newTestConsumer(sch, r, w, func(string) (Deduper, error) { return NewNoopDeduper(), nil })-- // deleted then completed: the completion is the last event for d1, so it- // wins — the doc is re-ingested, NOT deleted. The deletion must not drop- // its per-doc products, and the completion must be merged.- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeDeleted)); err != nil {- t.Fatalf("append deleted: %v", err)- }- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {- t.Fatalf("append completed: %v", err)- }- c.tryClaimAndProcess(context.Background())-- w.mu.Lock()- defer w.mu.Unlock()- // No deletion calls: the last (completion) event overrides the deletion.- if len(w.deletedDocLevel) != 0 {- t.Fatalf("expected no DeleteDocLevelForDocs, got %v", w.deletedDocLevel)- }- if len(w.strippedSources) != 0 {- t.Fatalf("expected no StripMergedSources, got %v", w.strippedSources)- }- // The completion is merged into the dataset-level products.- if len(w.written) != 1 {- t.Fatalf("expected 1 WriteMerged call, got %d", len(w.written))- }- if len(w.written[0]) != 2 {- t.Fatalf("expected 2 merged products, got %d", len(w.written[0]))- }- if len(w.projected) != 1 || w.projected[0] != "kb1" {- t.Fatalf("expected 1 ProjectWikiGraph(kb1) call, got %v", w.projected)- }-}--func TestSchedulerClaimClosedBatch(t *testing.T) {- sch := NewFakeScheduler()- for i := 0; i < 40; i++ {- docID := "d" + string(rune('a'+i%26)) + string(rune('0'+i/26))- if err := sch.Publish(context.Background(), "t1", "kb1", docID, string(EventTypeCompleted)); err != nil {- t.Fatalf("append: %v", err)- }- }- // First claim returns the bounded prefix (default batch=32), not all 40.- cr1, ok, err := sch.Claim(context.Background(), "kb1")- if err != nil || !ok {- t.Fatalf("claim1: ok=%v err=%v", ok, err)- }- if len(cr1.Entries) != 32 {- t.Fatalf("expected 32 entries in first claim, got %d", len(cr1.Entries))- }- // A second claim by the same holder (still live lease) must not re-claim- // the same dataset until the first batch is acked.- _, ok2, _ := sch.Claim(context.Background(), "kb1")- if ok2 {- t.Fatalf("second claim should have lost the race (live lease)")- }- // Ack the first batch, then the remaining 8 become claimable.- if _, err := sch.Ack(context.Background(), "kb1", cr1.Token, cr1.Entries); err != nil {- t.Fatalf("ack: %v", err)- }- cr3, ok3, err := sch.Claim(context.Background(), "kb1")- if err != nil || !ok3 {- t.Fatalf("claim3: ok=%v err=%v", ok3, err)- }- if len(cr3.Entries) != 8 {- t.Fatalf("expected 8 remaining entries, got %d", len(cr3.Entries))- }-}--func TestSchedulerReclaimExpired(t *testing.T) {- sch := NewFakeScheduler()- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {- t.Fatalf("append: %v", err)- }- _, ok, err := sch.Claim(context.Background(), "kb1")- if err != nil || !ok {- t.Fatalf("claim: ok=%v err=%v", ok, err)- }- // Simulate a crash: the inflight is never acked and the lease has expired.- past := time.Now().Add(-time.Hour)- sch.rows["kb1"].expires = &past- // TryClaim reclaims the expired lease back into backlog and immediately- // claims it again.- cr2, ok2, err := sch.TryClaim(context.Background())- if err != nil || !ok2 {- t.Fatalf("reclaim claim: ok=%v err=%v", ok2, err)- }- if len(cr2.Entries) != 1 {- t.Fatalf("expected 1 entry after reclaim, got %d", len(cr2.Entries))- }-}--// rowCounts snapshots a fake scheduling row's inflight/backlog entry counts.-type rowCounts struct {- inflight int- backlog int-}--func fakeRowCounts(sch *FakeScheduler, datasetID string) (rowCounts, string) {- sch.mu.Lock()- defer sch.mu.Unlock()- r := sch.rows[datasetID]- if r == nil {- return rowCounts{}, ""- }- return rowCounts{inflight: len(r.inflight), backlog: len(r.backlog)}, r.state-}--// TestSchedulerStateMachineLocksStateAndCounts locks the full lifecycle state-// machine (plan v4.1 §9.2) on the FakeScheduler, which mirrors the MySQL-// scheduler's transitions:-//-// Publish -> pending; Claim -> running (+error cleared);-// SetError (failed batch left for retry) keeps running + records error;-// lease expiry -> reclaimOne -> pending (inflight moved back to backlog);-// Ack with backlog drained -> completed.-func TestSchedulerStateMachineLocksStateAndCounts(t *testing.T) {- sch := NewFakeScheduler()-- // Publish two docs: state=pending, backlog=2, inflight=0.- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {- t.Fatalf("publish d1: %v", err)- }- if err := sch.Publish(context.Background(), "t1", "kb1", "d2", string(EventTypeCompleted)); err != nil {- t.Fatalf("publish d2: %v", err)- }- if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStatePending || c.backlog != 2 || c.inflight != 0 {- t.Fatalf("after publish: want state=pending backlog=2 inflight=0, got state=%s %+v", s, c)- }-- // Claim: state=running, backlog moves to inflight (batch=2), error cleared.- cr, ok, err := sch.Claim(context.Background(), "kb1")- if err != nil || !ok {- t.Fatalf("claim: ok=%v err=%v", ok, err)- }- if len(cr.Entries) != 2 {- t.Fatalf("expected 2-entry claim, got %d", len(cr.Entries))- }- if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStateRunning || c.backlog != 0 || c.inflight != 2 {- t.Fatalf("after claim: want state=running backlog=0 inflight=2, got state=%s %+v", s, c)- }-- // Failed batch left for retry: SetError records a diagnostic, state stays running.- if err := sch.SetError(context.Background(), "kb1", cr.Token, "boom"); err != nil {- t.Fatalf("set error: %v", err)- }- sch.mu.Lock()- gotErr := sch.rows["kb1"].errorMsg- sch.mu.Unlock()- if gotErr != "boom" {- t.Fatalf("expected errorMsg=boom, got %q", gotErr)- }- if _, s := fakeRowCounts(sch, "kb1"); s != DatasetStateRunning {- t.Fatalf("failed batch must stay running (left for retry), got state=%s", s)- }-- // Lease expires: reclaimOne moves inflight back to backlog, clears lease -> pending.- past := time.Now().Add(-time.Hour)- sch.mu.Lock()- sch.rows["kb1"].expires = &past- sch.mu.Unlock()- // Lock the reclaim transition in isolation (before any re-claim): the fake's- // reclaim helper is the same code path TryClaim uses, mirroring reclaimOne.- sch.mu.Lock()- if id := sch.fakeReclaimExpired(time.Now()); id != "kb1" {- sch.mu.Unlock()- t.Fatalf("expected kb1 to be reclaimed, got %q", id)- }- sch.mu.Unlock()- if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStatePending || c.backlog != 2 || c.inflight != 0 {- t.Fatalf("after reclaim: want state=pending backlog=2 inflight=0, got state=%s %+v", s, c)- }-- // Claim again then Ack the drained batch -> completed, counts zeroed.- cr2, ok3, err := sch.Claim(context.Background(), "kb1")- if err != nil || !ok3 {- t.Fatalf("re-claim: ok=%v err=%v", ok3, err)- }- if _, err := sch.Ack(context.Background(), "kb1", cr2.Token, cr2.Entries); err != nil {- t.Fatalf("ack: %v", err)- }- if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStateCompleted || c.backlog != 0 || c.inflight != 0 {- t.Fatalf("after ack drain: want state=completed backlog=0 inflight=0, got state=%s %+v", s, c)- }-}--// TestSchedulerSetErrorScopedToClaimToken locks the concurrency guard on-// SetError: a failed batch is diagnosed only while its own claim token is still-// live. If worker A's lease expires and is reclaimed, worker B re-claims and-// completes; a late SetError from A (stale token) must NOT overwrite the row's-// diagnostic, otherwise a completed state would be misread as failed.-func TestSchedulerSetErrorScopedToClaimToken(t *testing.T) {- sch := NewFakeScheduler()- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {- t.Fatalf("publish: %v", err)- }-- // A claims and begins processing (running).- crA, ok, err := sch.Claim(context.Background(), "kb1")- if err != nil || !ok {- t.Fatalf("claim A: ok=%v err=%v", ok, err)- }-- // A's lease expires before it finishes; the sweeper reclaims the inflight- // batch back to backlog (pending) and B takes over.- past := time.Now().Add(-time.Hour)- sch.mu.Lock()- sch.rows["kb1"].expires = &past- sch.mu.Unlock()- if id := sch.fakeReclaimExpired(time.Now()); id != "kb1" {- t.Fatalf("expected kb1 reclaimed, got %q", id)- }- crB, okB, err := sch.Claim(context.Background(), "kb1")- if err != nil || !okB {- t.Fatalf("claim B: ok=%v err=%v", okB, err)- }- if crA.Token == crB.Token {- t.Fatalf("expected distinct claim tokens, got %q", crA.Token)- }-- // B succeeds and drains the backlog to completed.- if _, err := sch.Ack(context.Background(), "kb1", crB.Token, crB.Entries); err != nil {- t.Fatalf("ack B: %v", err)- }- if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStateCompleted || c.backlog != 0 || c.inflight != 0 {- t.Fatalf("after B ack: want completed empty, got state=%s %+v", s, c)- }-- // A's late failure arrives with its now-stale token: it must be ignored.- if err := sch.SetError(context.Background(), "kb1", crA.Token, "stale failure from A"); err != nil {- t.Fatalf("stale set error: %v", err)- }- sch.mu.Lock()- gotErr := sch.rows["kb1"].errorMsg- sch.mu.Unlock()- if gotErr != "" {- t.Fatalf("stale SetError overwrote diagnostic: got errorMsg=%q want empty", gotErr)- }- if _, s := fakeRowCounts(sch, "kb1"); s != DatasetStateCompleted {- t.Fatalf("stale SetError changed state: got %q want completed", s)- }-}diff --git a/internal/ingestion/knowledge_compile/dedup.go b/internal/ingestion/knowledge_compile/dedup.goindex 3018b4b157..37785f6481 100644--- a/internal/ingestion/knowledge_compile/dedup.go+++ b/internal/ingestion/knowledge_compile/dedup.go@@ -111,27 +111,57 @@ func (x *llmDeduper) Decide(ctx context.Context, existing, incoming kccommon.Proreturn kccommon.Product{}, false, nil}-// DecideBatch folds every group with a single LLM call. All candidate pairs-// across all groups are judged at once (mergePairsBatch); each group then folds-// its candidates into its existing row in order so a chain of merges within a-// group accumulates correctly.+// DecideBatch folds every group into its dataset-level merged row. Wiki groups+// use a REPLACE-ONLY strategy (wikiDecideBatch) that never invokes the LLM JSON+// merge — Markdown wiki pages must not be concatenated/merged by the generic+// structure decider. Structure (and other non-wiki) groups are judged by the LLM+// merge decider as before.+// splitWikiGroups partitions the batch into wiki groups (replace-only, no LLM)+// and structure groups (LLM-merged). structIdx[i] is the ORIGINAL position in+// `groups` of structGroups[i]; it is what the fold uses to write results back to+// the right slice element. Recording the position inside structGroups (i.e.+// len(structGroups)) instead would always equal i and misroute structure results+// to the wrong groups whenever a wiki group appears earlier in the batch.+func splitWikiGroups(groups []MergeGroup) (wikiIdx, structIdx []int, structGroups []MergeGroup) {+ for gi := range groups {+ if isWikiGroup(groups[gi]) {+ wikiIdx = append(wikiIdx, gi)+ } else {+ structIdx = append(structIdx, gi)+ structGroups = append(structGroups, groups[gi])+ }+ }+ return wikiIdx, structIdx, structGroups+}+func (x *llmDeduper) DecideBatch(ctx context.Context, groups []MergeGroup) ([]MergeGroup, error) {- // Assign a flat pair index to every (group, candidate).+ // Split into wiki (replace-only, no LLM) and structure (LLM-merged) groups.+ wikiIdx, structIdx, structGroups := splitWikiGroups(groups)+ // Wiki groups: replace-only, in place.+ for _, gi := range wikiIdx {+ groups[gi] = wikiDecideBatch(ctx, []MergeGroup{groups[gi]})[0]+ }+ if len(structGroups) == 0 {+ return groups, nil+ }++ // Assign a flat pair index to every (group, candidate) of the structure groups.var inputs []structure.MergePairInput- pairIndexOf := make([][]int, len(groups))- for gi := range groups {- pairIndexOf[gi] = make([]int, len(groups[gi].Candidates))- for ci := range groups[gi].Candidates {+ pairIndexOf := make([][]int, len(structGroups))+ for gi := range structGroups {+ pairIndexOf[gi] = make([]int, len(structGroups[gi].Candidates))+ for ci := range structGroups[gi].Candidates {idx := len(inputs)pairIndexOf[gi][ci] = idxinputs = append(inputs, structure.MergePairInput{Index: idx,- Existing: groups[gi].Existing.Content,- Incoming: groups[gi].Candidates[ci].Content,+ Existing: structGroups[gi].Existing.Content,+ Incoming: structGroups[gi].Candidates[ci].Content,})}}if len(inputs) == 0 {+ // Only wiki groups had candidates; copy them back (already folded above).return groups, nil}results, err := x.decider.DecideBatch(ctx, inputs)@@ -143,12 +173,12 @@ func (x *llmDeduper) DecideBatch(ctx context.Context, groups []MergeGroup) ([]MebyIndex[r.Index] = r}- for gi := range groups {- existing := groups[gi].Existing+ for si, gi := range structIdx {+ existing := structGroups[si].Existingvar distinct []kccommon.Productduplicated := false- for ci, cand := range groups[gi].Candidates {- r := byIndex[pairIndexOf[gi][ci]]+ for ci, cand := range structGroups[si].Candidates {+ r := byIndex[pairIndexOf[si][ci]]if !r.Duplicated || r.Merged == nil {// Judged distinct: keep it as its own new merged row.c := canddiff --git a/internal/ingestion/knowledge_compile/dedup_test.go b/internal/ingestion/knowledge_compile/dedup_test.goindex a4a178356d..c90193401d 100644--- a/internal/ingestion/knowledge_compile/dedup_test.go+++ b/internal/ingestion/knowledge_compile/dedup_test.go@@ -97,6 +97,47 @@ func TestDeduperDecideBatchFoldsGroups(t *testing.T) {}}+// TestSplitWikiGroupsMapsStructureIndexes locks the Critical fix in+// llmDeduper.DecideBatch: structIdx[i] must hold the ORIGINAL position in the+// batch, not the position inside the structure-only slice. When a wiki group+// appears before a structure group, a buggy len(structGroups) mapping would make+// structIdx[0]==0 and fold structure results back into the wiki group instead of+// the structure group at index 1.+func TestSplitWikiGroupsMapsStructureIndexes(t *testing.T) {+ groups := []MergeGroup{+ {Existing: kccommon.Product{ID: "wiki", Variant: kccommon.VariantWiki, Content: "w"}},+ {Existing: kccommon.Product{ID: "struct-0", Variant: kccommon.Variant("structure"), Content: "s0"}},+ {Existing: kccommon.Product{ID: "wiki-2", Variant: kccommon.VariantWiki, Content: "w2"}},+ {Existing: kccommon.Product{ID: "struct-1", Variant: kccommon.Variant("structure"), Content: "s1"}},+ }+ wikiIdx, structIdx, structGroups := splitWikiGroups(groups)++ if len(wikiIdx) != 2 || wikiIdx[0] != 0 || wikiIdx[1] != 2 {+ t.Fatalf("wikiIdx = %v, want [0 2]", wikiIdx)+ }+ if len(structIdx) != 2 {+ t.Fatalf("structIdx = %v, want two structure entries", structIdx)+ }+ // The structure groups appear at original batch indices 1 and 3.+ wantStructIdx := []int{1, 3}+ for i, gi := range structIdx {+ if gi != wantStructIdx[i] {+ t.Fatalf("structIdx[%d] = %d, want %d (original batch index)", i, gi, wantStructIdx[i])+ }+ // groups[structIdx[i]] must be the exact element copied into structGroups[i],+ // so folding back via groups[structIdx[i]] lands on the correct group.+ if groups[gi].Existing.ID != structGroups[i].Existing.ID {+ t.Fatalf("groups[%d] (%s) is not the same element as structGroups[%d] (%s)",+ gi, groups[gi].Existing.ID, i, structGroups[i].Existing.ID)+ }+ }+ // The buggy mapping would have produced structIdx == [0 1], pointing at the+ // wiki groups; assert we did NOT regress to that.+ if structIdx[0] == 0 || structIdx[1] == 1 {+ t.Fatalf("structIdx mapped to the wiki groups' positions: %v", structIdx)+ }+}+func TestNoopDeduperDecideBatchNoMerge(t *testing.T) {d := NewNoopDeduper()existing := kccommon.Product{ID: "row-1", DocID: "kb1", Content: "base"}diff --git a/internal/ingestion/knowledge_compile/fake_engine_test.go b/internal/ingestion/knowledge_compile/fake_engine_test.gonew file mode 100644index 0000000000..48fe8e98ef--- /dev/null+++ b/internal/ingestion/knowledge_compile/fake_engine_test.go@@ -0,0 +1,114 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package knowledge_compile++import (+ "context"++ "ragflow/internal/engine/types"++ "gorm.io/gorm"+)++// fakeEngine is a minimal engine.DocEngine for unit tests. It records the+// Search filter/SelectFields and the DeleteChunks condition so tests can assert+// the structural scope of a reader/writer call, and returns a canned chunk set.+type fakeEngine struct {+ // searchChunks is the result returned by Search.+ searchChunks []map[string]interface{}+ // lastSearchReq captures the most recent SearchRequest for assertions.+ lastSearchReq *types.SearchRequest+ // lastDeleteCond captures the most recent DeleteChunks condition.+ lastDeleteCond map[string]interface{}+ // deleteCount is the number returned by DeleteChunks.+ deleteCount int64+}++func (f *fakeEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ f.lastSearchReq = req+ return &types.SearchResult{Chunks: f.searchChunks}, nil+}++func (f *fakeEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, _, _ string) (int64, error) {+ f.lastDeleteCond = condition+ return f.deleteCount, nil+}++// --- stubs: the reader/writer path under test never exercises these. ---++func (f *fakeEngine) CreateChunkStore(context.Context, string, string, int, string) error {+ return nil+}+func (f *fakeEngine) InsertChunks(context.Context, []map[string]interface{}, string, string) ([]string, error) {+ return nil, nil+}+func (f *fakeEngine) UpdateChunks(context.Context, map[string]interface{}, map[string]interface{}, string, string) error {+ return nil+}+func (f *fakeEngine) GetChunk(context.Context, string, string, []string) (interface{}, error) {+ return nil, nil+}+func (f *fakeEngine) DropChunkStore(context.Context, string, string) error { return nil }+func (f *fakeEngine) ChunkStoreExists(context.Context, string, string) (bool, error) {+ return true, nil+}+func (f *fakeEngine) CreateMetadataStore(context.Context, string) error { return nil }+func (f *fakeEngine) InsertMetadata(context.Context, []map[string]interface{}, string) ([]string, error) {+ return nil, nil+}+func (f *fakeEngine) UpdateMetadata(context.Context, string, string, map[string]interface{}, string) error {+ return nil+}+func (f *fakeEngine) DeleteMetadata(context.Context, map[string]interface{}, string) (int64, error) {+ return 0, nil+}+func (f *fakeEngine) DeleteMetadataKeys(context.Context, string, string, []string, string) error {+ return nil+}+func (f *fakeEngine) DropMetadataStore(context.Context, string) error { return nil }+func (f *fakeEngine) MetadataStoreExists(context.Context, string) (bool, error) { return true, nil }+func (f *fakeEngine) SearchMetadata(context.Context, *types.SearchMetadataRequest) (*types.SearchMetadataResult, error) {+ return nil, nil+}+func (f *fakeEngine) IndexDocument(context.Context, string, string, interface{}) error { return nil }+func (f *fakeEngine) DeleteDocument(context.Context, string, string) error { return nil }+func (f *fakeEngine) BulkIndex(context.Context, string, []interface{}) (interface{}, error) {+ return nil, nil+}+func (f *fakeEngine) GetFields([]map[string]interface{}, []string) map[string]map[string]interface{} {+ return nil+}+func (f *fakeEngine) GetAggregation([]map[string]interface{}, string) []map[string]interface{} {+ return nil+}+func (f *fakeEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {+ return nil+}+func (f *fakeEngine) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {+ return nil, nil+}+func (f *fakeEngine) GetChunkIDs([]map[string]interface{}) []string { return nil }+func (f *fakeEngine) KNNScores(context.Context, []map[string]interface{}, []float64, int) (map[string]interface{}, error) {+ return nil, nil+}+func (f *fakeEngine) GetScores(map[string]interface{}) map[string]float64 { return nil }+func (f *fakeEngine) Ping(context.Context) error { return nil }+func (f *fakeEngine) Close() error { return nil }+func (f *fakeEngine) GetType() string { return "fake" }+func (f *fakeEngine) SupportsPageRank() bool { return false }+func (f *fakeEngine) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []string, []map[string]interface{}, string) []string {+ return nil+}diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 7141fe6c57..809605b86f 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -63,6 +63,14 @@ type engineReader struct {// compiledSelectFields are the columns needed to reconstruct a Product from a// stored compiled chunk document.+//+// wiki_incremental port: the list also selects `kc_kind` and+// `create_timestamp_flt` (+`create_time`) so the reader can round-trip the+// wiki product kind (page/section) and the original creation timestamp without+// re-deriving them (see productFromChunkMap). Without these in the SELECT list,+// the stored values are invisible to the reader and every merged row would fall+// back to the compile_kwd-derived kind / a fresh now() timestamp — which both+// breaks the page/section filter and re-stamps creation time on every rebuild.var compiledSelectFields = []string{"id", "doc_id", "tenant_id", "compile_kwd","available_int",@@ -70,6 +78,7 @@ var compiledSelectFields = []string{"source_chunk_ids", "source_doc_ids","name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd","slug_kwd", "type",+ "kc_kind", "create_timestamp_flt", "create_time",}// wikiSelectFields are the additional columns a wiki page carries (beyond@@ -126,7 +135,13 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID strif _, ok := c["compile_kwd"]; !ok {continue}- if p, ok := productFromChunkMap(c, tenant); ok {+ // Reverse-map the row's compile_kwd; reject dirty/unknown kinds+ // (kwdToVariant error) so a malformed row never loads as a product.+ rowVariant, verr := kwdToVariant(asString(c["compile_kwd"]))+ if verr != nil {+ continue+ }+ if p, ok := productFromChunkMap(c, tenant, rowVariant); ok {out = append(out, p)}}@@ -159,7 +174,14 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID str// productFromChunkMap reconstructs a kccommon.Product from a stored compiled// chunk document. It reads the payload from kc_payload (falling back to// content_with_weight) and the embedding from the q_<dim>_vec column.-func productFromChunkMap(c map[string]interface{}, tenant string) (kccommon.Product, bool) {+//+// expect is the variant the caller is querying for. The stored compile_kwd is+// reverse-mapped via kwdToVariant and compared against expect; a mismatch (or+// an unknown/dirty compile_kwd that does not map to any known variant) causes+// the row to be skipped. This is the canonical dirty-row contract promised by+// the plan: we never rely on the raw string equality alone, so unknown kinds+// are rejected consistently rather than leaking into the wrong bucket.+func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommon.Variant) (kccommon.Product, bool) {content, _ := c["kc_payload"].(string)if content == "" {content, _ = c["content_with_weight"].(string)@@ -169,7 +191,17 @@ func productFromChunkMap(c map[string]interface{}, tenant string) (kccommon.Prod}id, _ := c["id"].(string)docID, _ := c["doc_id"].(string)- variant, _ := c["compile_kwd"].(string)+ // Normalize through the shared asString helper (matching LoadDocProducts) so+ // a non-string scalar or a list-wrapped keyword column from the engine is+ // reverse-mapped consistently instead of being rejected as a dirty row.+ variant := asString(c["compile_kwd"])+ // Dirty-row contract: the raw compile_kwd must reverse-map to the expected+ // variant. An unknown/dirty kwd (kwdToVariant error) or a mapped variant+ // that differs from expect is rejected.+ mapped, err := kwdToVariant(variant)+ if err != nil || mapped != expect {+ return kccommon.Product{}, false+ }merged := isAvailable(c["available_int"])meta := map[string]any{}@@ -226,6 +258,30 @@ func productFromChunkMap(c map[string]interface{}, tenant string) (kccommon.Prodmeta["kind"] = "entity"}}+ // wiki_incremental port: round-trip the wiki product kind so the+ // dataset-level merge can reliably distinguish pages from sections. The+ // merged writer stores kc_kind; when present it is authoritative. Without+ // it (legacy rows), derive from compile_kwd: wiki_page -> "page",+ // wiki_section -> "section". This fix is what stops the processBatch+ // "Meta.kind==page" filter from deleting every wiki page (previously kind+ // was empty for wiki pages that had no entity/relation endpoint).+ if v, ok := c["kc_kind"].(string); ok && v != "" {+ meta["kind"] = v+ } else if variant == compileKwdWikiPage {+ meta["kind"] = "page"+ } else if variant == compileKwdWikiSection {+ meta["kind"] = "section"+ }+ // wiki_incremental port: restore the original creation timestamp so a+ // replace-only merge (wikiMerge.Replace) preserves it instead of stamping+ // a fresh now(). existing rows carry create_timestamp_flt (and optionally+ // a human-readable create_time string).+ if v, ok := metaFloat(c, "create_timestamp_flt"); ok {+ meta["created_at_unix"] = v+ }+ if v, ok := c["create_time"].(string); ok && v != "" {+ meta["created_at"] = v+ }if v := metaStringSlice(c, "source_chunk_ids"); len(v) > 0 {meta["source_chunk_ids"] = v}@@ -238,7 +294,7 @@ func productFromChunkMap(c map[string]interface{}, tenant string) (kccommon.ProdID: id,DocID: docID,TenantID: tenant,- Variant: kccommon.Variant(variant),+ Variant: expect,Content: content,Vector: vec,Meta: meta,@@ -266,11 +322,12 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, variLimit: topN,SelectFields: append([]string{"id", "doc_id", "kb_id", "content_with_weight", "kc_payload","name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd", "slug_kwd",- "type", "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd"},+ "type", "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd",+ "kc_kind", "create_timestamp_flt", "create_time"},wikiSelectFields...),Filter: map[string]interface{}{"available_int": 1,- "compile_kwd": string(variant),+ "compile_kwd": compileKwdForVariant(variant),},MatchExprs: []interface{}{&types.MatchDenseExpr{@@ -286,8 +343,23 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, variif err != nil {return kccommon.Product{}, 0, err}+ // The KNN Filter scopes compile_kwd, but a foreign/legacy row that slipped+ // past it must not poison the merge candidate. Reject by the raw compile_kwd+ // (wiki_page vs wiki_section are distinct keywords even though both map to+ // VariantWiki); the page/section distinction is resolved downstream by+ // Meta.kind (see productFromChunkMap).+ expectKwd := compileKwdForVariant(variant)for _, c := range res.Chunks {- p, ok := productFromChunkMap(c, tenant)+ // The KNN Filter already scopes compile_kwd, but a foreign/legacy row that+ // slipped past it must not poison the merge candidate. Reject by the+ // reverse-mapped variant (the dirty-row contract): productFromChunkMap+ // validates kwdToVariant(c) == variant and drops dirty/unknown kinds. The+ // raw-keyword check below is a fast pre-filter before the full product+ // reconstruction.+ if asString(c["compile_kwd"]) != expectKwd {+ continue+ }+ p, ok := productFromChunkMap(c, tenant, variant)if !ok || !p.Merged {continue}@@ -335,6 +407,36 @@ func isAvailable(v interface{}) bool {return false}+// asString normalizes a boxed chunk-map value into a string (used where the+// backend may return a typed string, a json.Number, or a single-element list for+// a keyword column). A list-wrapped keyword (e.g. []string{"wiki_page"} or+// []any{"wiki_page"}) is unwrapped to its first element so reverse-mapping and+// raw-keyword filters behave consistently across engine backends.+func asString(v interface{}) string {+ switch t := v.(type) {+ case nil:+ return ""+ case string:+ return t+ case json.Number:+ return t.String()+ case []string:+ if len(t) == 1 {+ return t[0]+ }+ return ""+ case []interface{}:+ if len(t) == 1 {+ if s, ok := t[0].(string); ok {+ return s+ }+ return fmt.Sprintf("%v", t[0])+ }+ return ""+ }+ return fmt.Sprintf("%v", v)+}+// toFloat64 normalizes the boxed score field returned by the DocEngine into a// float64, accepting float32, float64, numeric strings, and json.Number. It// returns 0 when the value is missing or not numeric.diff --git a/internal/ingestion/knowledge_compile/reader_test.go b/internal/ingestion/knowledge_compile/reader_test.gonew file mode 100644index 0000000000..9642e1a89d--- /dev/null+++ b/internal/ingestion/knowledge_compile/reader_test.go@@ -0,0 +1,125 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package knowledge_compile++import (+ "context"+ "testing"++ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// TestSearchSimilarFiltersByVariant asserts the B1/KNN contract: SearchSimilar+// scopes the engine query to available_int=1 AND compile_kwd=variant, and the+// in-memory dirty-row guard drops any row whose compile_kwd does not map back to+// the variant (so a foreign row can never become the merge candidate).+func TestSearchSimilarFiltersByVariant(t *testing.T) {+ eng := &fakeEngine{+ searchChunks: []map[string]interface{}{+ // dirty row: wrong variant (must be skipped by the in-memory guard).+ {+ "id": "dirty",+ "doc_id": "kb",+ "available_int": 1,+ "compile_kwd": "wiki_section", // not wiki_page+ "kc_payload": "{\"c\":1}",+ "_score": 0.99,+ },+ // good row: correct variant + kc_kind round-trip.+ {+ "id": "page1",+ "doc_id": "kb",+ "available_int": 1,+ "compile_kwd": "wiki_page",+ "kc_kind": "page",+ "kc_payload": "{\"c\":1}",+ "create_timestamp_flt": 1700000000.0,+ "create_time": "2023-11-14T22:13:20Z",+ "_score": 0.95,+ },+ },+ }+ r := engineReader{eng: eng}++ p, score, err := r.SearchSimilar(context.Background(), "t1", "kb", kccommon.VariantWiki, []float64{0.1, 0.2, 0.3}, 2, 0.5)+ if err != nil {+ t.Fatalf("SearchSimilar: %v", err)+ }+ if p.ID != "page1" {+ t.Fatalf("expected the wiki_page row to win, got %q", p.ID)+ }+ if p.Meta["kind"] != "page" {+ t.Fatalf("expected kc_kind round-trip to 'page', got %v", p.Meta["kind"])+ }+ if p.Merged != true {+ t.Fatalf("expected merged=true for available_int=1 row")+ }+ if _, ok := p.Meta["created_at_unix"]; !ok {+ t.Fatalf("expected created_at_unix round-trip from create_timestamp_flt")+ }+ if score != 0.95 {+ t.Fatalf("expected _score=0.95, got %v", score)+ }++ // Structural filter: the engine request must scope to the variant.+ if eng.lastSearchReq == nil {+ t.Fatal("engine.Search was not called")+ }+ if eng.lastSearchReq.Filter["available_int"] != 1 {+ t.Fatalf("expected available_int=1 filter, got %v", eng.lastSearchReq.Filter["available_int"])+ }+ if eng.lastSearchReq.Filter["compile_kwd"] != string(compileKwdWikiPage) {+ t.Fatalf("expected compile_kwd=%q filter, got %v", compileKwdWikiPage, eng.lastSearchReq.Filter["compile_kwd"])+ }+ // SelectFields must include the round-trip columns added in the 8th review.+ for _, want := range []string{"kc_kind", "create_timestamp_flt", "create_time"} {+ found := false+ for _, f := range eng.lastSearchReq.SelectFields {+ if f == want {+ found = true+ break+ }+ }+ if !found {+ t.Fatalf("SearchSimilar SelectFields missing %q", want)+ }+ }+}++// TestSearchSimilarSkipsNonMerged asserts a row with available_int=0 (a+// per-document row, not a dataset-level merged row) is never a KNN candidate.+func TestSearchSimilarSkipsNonMerged(t *testing.T) {+ eng := &fakeEngine{+ searchChunks: []map[string]interface{}{+ {+ "id": "doc1",+ "doc_id": "doc",+ "available_int": 0,+ "compile_kwd": "wiki_page",+ "kc_payload": "{\"c\":1}",+ "_score": 0.99,+ },+ },+ }+ r := engineReader{eng: eng}+ p, _, err := r.SearchSimilar(context.Background(), "t1", "kb", kccommon.VariantWiki, []float64{0.1}, 1, 0.5)+ if err != nil {+ t.Fatalf("SearchSimilar: %v", err)+ }+ if p.ID != "" {+ t.Fatalf("expected no candidate from a non-merged row, got %q", p.ID)+ }+}diff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.goindex 479d285279..bf9d7f136b 100644--- a/internal/ingestion/knowledge_compile/scheduler.go+++ b/internal/ingestion/knowledge_compile/scheduler.go@@ -114,6 +114,26 @@ type Claimer interface {// status of the worker that took over.SetError(ctx context.Context, datasetID, token, errMsg string) error+ // CancelInflight drops the live claim for a dataset back into the backlog so+ // a rewrite can start from a clean index. The token is intentionally NOT+ // checked: this is the rewrite's authoritative cancel. Workers verify their+ // claim token inside the dataset write lock before every destructive write, so+ // a cancelled worker cannot write after the rebuild clears storage. It is a safe no-op when+ // there is no live lease (the dataset is idle or already drained). The dropped+ // batch is re-marked pending (not deleted) so it is reclaimed and re-checked+ CancelInflight(ctx context.Context, datasetID, token string) error++ // WithDatasetLock executes fn while holding an exclusive per-dataset lock+ // that serializes against both writer side effects and dataset rebuilds+ // (RebuildDataset). It is the synchronization that closes the TOCTOU gap+ // between a worker's claim validation and its destructive write: the caller+ // must compare its claim token and perform the write inside fn. fn+ // runs inside a transaction that holds the dataset row lock for its whole+ // duration (MySQL) or a per-row mutex (Fake), so rebuild and writes are+ // mutually exclusive. claimToken is read under the lock; the caller must not+ // re-read it on a separate connection.+ WithDatasetLock(ctx context.Context, datasetID string, fn func(claimToken string) error) error+// SubscribeNotify returns a channel of dataset ids pushed by Publish, or nil// when the implementation has no push wake-up (callers fall back to polling).SubscribeNotify(ctx context.Context) (<-chan string, error)@@ -184,6 +204,23 @@ func (s *mysqlScheduler) Provision(ctx context.Context) error {// each other's backlog), and the notify is always paired with the append so a// producer never needs a separate Notify call.func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error {+ return s.publish(ctx, tenantID, datasetID, docID, eventType)+}++// loadRow fetches the dataset scheduling row (no lock). Returns nil (no error)+// when the row does not exist yet.+func (s *mysqlScheduler) loadRow(ctx context.Context, datasetID string) (*entity.KnowledgeCompileDataset, error) {+ var row entity.KnowledgeCompileDataset+ if err := s.db.WithContext(ctx).Where("dataset_id = ?", datasetID).First(&row).Error; err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return nil, nil+ }+ return nil, err+ }+ return &row, nil+}++func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error {if s.db == nil {return nil}@@ -235,6 +272,39 @@ func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docIDreturn s.notify(ctx, datasetID)}+// WithDatasetLock executes fn under an exclusive per-dataset row lock. It locks+// the dataset row (SELECT ... FOR UPDATE) and runs fn inside the same+// transaction, so the row lock is held for the entire duration of fn and any+// concurrent WithDatasetLock on the same dataset blocks until fn returns. This+// is what makes a worker's claim-token check + destructive write atomic against+// a concurrent RebuildDataset.+//+// The claim token passed to fn is read within this transaction, so the check in+// fn does not need (and must not attempt) a separate DB read that would deadlock+// against the held row lock. fn runs inside the transaction; if it returns an+// error the transaction is rolled back+// and the error is propagated unchanged (any DB write performed inside fn is+// rolled back, so no partial side effect persists on an aborted write).+func (s *mysqlScheduler) WithDatasetLock(ctx context.Context, datasetID string, fn func(claimToken string) error) error {+ if s.db == nil {+ // No DB (test/no-op scheduler): nothing to serialize against.+ return fn("")+ }+ return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ row := entity.KnowledgeCompileDataset{+ DatasetID: datasetID,+ BacklogDocIDs: "[]",+ InflightDocIDs: "[]",+ }+ if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).+ Where("dataset_id = ?", datasetID).+ FirstOrCreate(&row).Error; err != nil {+ return err+ }+ return fn(row.ClaimToken)+ })+}+// claimRow atomically claims the closed batch from the row identified by// datasetID: it takes a FOR UPDATE row lock, refuses a live lease, moves up to// claimBatchSize entries from backlog to inflight, and stamps the lease. The@@ -381,6 +451,46 @@ func (s *mysqlScheduler) SetError(ctx context.Context, datasetID, token, errMsgreturn nil}+// CancelInflight drops the live claim for a dataset back into the backlog so a+// rewrite can start from a clean index. It is a no-op when there is no live+// lease. The dropped batch is re-marked pending and is reprocessed after the+// rebuild republishes the dataset documents.+func (s *mysqlScheduler) CancelInflight(ctx context.Context, datasetID, token string) error {+ if s.db == nil {+ return nil+ }+ err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ var row entity.KnowledgeCompileDataset+ if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}).+ Where("dataset_id = ?", datasetID).First(&row).Error; err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return nil+ }+ return err+ }+ // No live lease (idle or already drained) → nothing to cancel.+ now := time.Now()+ liveLease := row.ClaimOwner != "" && row.ClaimExpiresAt != nil && row.ClaimExpiresAt.After(now)+ if !liveLease {+ return nil+ }+ inflight := parseEntries(row.InflightDocIDs)+ backlog := parseEntries(row.BacklogDocIDs)+ backlog = append(backlog, inflight...)+ row.InflightDocIDs = "[]"+ row.ClaimOwner = ""+ row.ClaimToken = ""+ row.ClaimExpiresAt = nil+ row.BacklogDocIDs = marshalEntries(backlog)+ row.State = DatasetStatePending+ return tx.Save(&row).Error+ })+ if err != nil {+ return fmt.Errorf("knowledge_compile: cancel inflight %s: %w", datasetID, err)+ }+ return nil+}+func (s *mysqlScheduler) TouchClaim(ctx context.Context, datasetID, token string, ttl time.Duration) (bool, error) {if s.db == nil {return false, nil@@ -561,6 +671,9 @@ type fakeRow struct {expires *time.Timestate stringerrorMsg string+ // writeMu serializes per-dataset writer side effects and rebuilds, mirroring+ // the MySQL scheduler's WithDatasetLock row lock.+ writeMu sync.Mutex}// FakeScheduler is an in-memory Publisher + Claimer used by tests. It mirrors@@ -585,8 +698,12 @@ func NewFakeScheduler() *FakeScheduler {func (f *FakeScheduler) Provision(_ context.Context) error { return nil }-// Publish appends one doc event and pushes a notify (same as the MySQL path).+// Publish appends one doc event and pushes a notify (same contract as MySQL).func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string) error {+ return f.publish(tenantID, datasetID, docID, eventType)+}++func (f *FakeScheduler) publish(tenantID, datasetID, docID, eventType string) error {f.mu.Lock()defer f.mu.Unlock()r, ok := f.rows[datasetID]@@ -683,6 +800,47 @@ func (f *FakeScheduler) SetError(_ context.Context, datasetID, token, errMsg strreturn nil}+// WithDatasetLock executes fn under the per-row write mutex, mirroring the MySQL+// scheduler's row-lock semantics: a rebuild and a writer's destructive side+// effect on the same dataset are mutually exclusive. claimToken is read under+// the lock.+func (f *FakeScheduler) WithDatasetLock(_ context.Context, datasetID string, fn func(claimToken string) error) error {+ f.mu.Lock()+ r, ok := f.rows[datasetID]+ if !ok {+ r = &fakeRow{}+ f.rows[datasetID] = r+ }+ f.mu.Unlock()++ r.writeMu.Lock()+ defer r.writeMu.Unlock()+ f.mu.Lock()+ claimToken := r.token+ f.mu.Unlock()+ return fn(claimToken)+}++// CancelInflight drops the live claim back into the backlog (mirrors MySQL).+func (f *FakeScheduler) CancelInflight(_ context.Context, datasetID, _ string) error {+ f.mu.Lock()+ defer f.mu.Unlock()+ r, ok := f.rows[datasetID]+ if !ok {+ return nil+ }+ now := time.Now()+ live := r.owner != "" && r.expires != nil && r.expires.After(now)+ if !live {+ return nil+ }+ r.backlog = append(r.backlog, r.inflight...)+ r.inflight = nil+ r.owner, r.token, r.expires = "", "", nil+ r.state = DatasetStatePending+ return nil+}+// fakeReclaimExpired finds one dataset with an expired lease and non-empty// inflight, moves the inflight batch back to backlog, clears the lease, and// marks the row pending — mirroring the MySQL scheduler's reclaimOne. It returnsdiff --git a/internal/ingestion/knowledge_compile/service.go b/internal/ingestion/knowledge_compile/service.goindex 4acbb0975a..6377a38269 100644--- a/internal/ingestion/knowledge_compile/service.go+++ b/internal/ingestion/knowledge_compile/service.go@@ -123,6 +123,7 @@ func Provision(ctx context.Context, mq engine.MessageQueue, db *gorm.DB) error {if db == nil {return nil}+ kcDB = dbs := newScheduler(db, mq, generateHolder(), 2*time.Minute)// Bound the startup AutoMigrate so a slow/unreachable DB cannot block// startup indefinitely. The caller's ctx is also honoured (cancelled ondiff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.goindex 0fc55eacaf..a0e39e3c82 100644--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go@@ -156,6 +156,45 @@ func TestProjectWikiGraphRowsWeightEqualsOutlinkCount(t *testing.T) {}}+// TestProjectWikiGraphRowsBareSlugOutlinkCompletesGraph asserts the G1 reader+// parity fix: an outlink given as a bare slug ("beta", not "entity/beta") is+// resolved to the full slug via the page slug map and still yields a relation,+// so the wiki nav graph is complete (matching Python dataset_wiki_generator).+func TestProjectWikiGraphRowsBareSlugOutlinkCompletesGraph(t *testing.T) {+ w := engineWriter{}+ pages := []wikiPageProjection{+ page("entity", "alpha", "beta"), // bare outlink resolves to entity/beta+ page("entity", "beta"),+ }+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)+ if err != nil {+ t.Fatalf("project: %v", err)+ }+ if findRelation(rows, "entity/alpha", "entity/beta") == nil {+ t.Fatalf("bare-slug outlink must resolve to a relation entity/alpha->entity/beta")+ }+}++// TestProjectWikiGraphRowsDropsDanglingAndSelfLoop asserts a dangling outlink+// (target page absent) and a self-loop (alpha->alpha) are skipped, so the graph+// never references non-existent vertices.+func TestProjectWikiGraphRowsDropsDanglingAndSelfLoop(t *testing.T) {+ w := engineWriter{}+ pages := []wikiPageProjection{+ page("entity", "alpha", "alpha", "ghost"), // self-loop + dangling+ }+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)+ if err != nil {+ t.Fatalf("project: %v", err)+ }+ if findRelation(rows, "entity/alpha", "entity/alpha") != nil {+ t.Fatalf("self-loop must be skipped")+ }+ if findRelation(rows, "entity/alpha", "entity/ghost") != nil {+ t.Fatalf("dangling outlink must be skipped")+ }+}+// TestProjectWikiGraphRowsFieldMapping verifies the reader-facing display// columns are written (aliases_kwd from entity_names_kwd, description_with_weight// from summary_with_weight), and the id is the xxhash colon-namespaced form.diff --git a/internal/ingestion/knowledge_compile/wiki_merge.go b/internal/ingestion/knowledge_compile/wiki_merge.gonew file mode 100644index 0000000000..f86e2b0cc3--- /dev/null+++ b/internal/ingestion/knowledge_compile/wiki_merge.go@@ -0,0 +1,156 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package knowledge_compile++import (+ "context"++ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// wikiReplace folds a wiki page candidate into its existing dataset-level merged+// row using a REPLACE-ONLY strategy. Wiki pages are Markdown, so the generic+// structure.LLMMergeDecider JSON merge (which concatenates/merges arbitrary JSON+// payloads) would mangle the page body — it must never run on wiki content.+//+// The existing row keeps its identity (id / doc_id / creation time) and provenance+// is unioned (source_doc_ids + source_chunk_ids of existing ∪ candidate). The+// page body is replaced by the candidate's Markdown verbatim. This mirrors+// Python's wiki incremental mode where a page re-compiled from a newer document+// simply supersedes the stored page of the same slug, rather than being merged.+func wikiReplace(existing, candidate kccommon.Product) kccommon.Product {+ merged := existing++ // Body: the newer (incoming) Markdown wins verbatim.+ merged.Content = candidate.Content+ // The embedding must match the replacement Markdown. WriteMerged persists+ // p.Vector without re-embedding, so keeping existing.Vector would leave a+ // stale embedding for the old page body and break KNN similarity searches.+ merged.Vector = candidate.Vector++ // Identity preserved: keep existing id, doc_id (== kb), and the original+ // creation timestamp (carried via Meta.created_at_unix by the Reader).+ merged.ID = existing.ID+ merged.DocID = existing.DocID++ // Provenance union (deduped) so the merged page references every source doc+ // and chunk that contributed to it across runs.+ merged.Meta = unionWikiProvenance(existing.Meta, candidate.Meta)++ // Re-stamp the resolver/run id from the incoming candidate so the row is+ // attributed to the latest batch, but the created_at_unix stays with existing.+ if v, ok := candidate.Meta["run_id"]; ok {+ merged.Meta["run_id"] = v+ }+ return merged+}++// unionWikiProvenance returns a new Meta map based on a (the existing row). The+// candidate b overwrites kind / slug / page_type / title / summary /+// entity_names / related_kb_pages / outlinks. Identity and creation time+// (created_at_unix, created_at) stay from a. Source provenance arrays+// (source_doc_ids, source_chunk_ids) are unioned and deduped.+func unionWikiProvenance(a, b map[string]any) map[string]any {+ out := map[string]any{}+ for k, v := range a {+ out[k] = v+ }+ // The incoming page replaces current page metadata. Identity and creation+ // time remain from the existing row.+ for _, key := range []string{"slug", "page_type", "title", "summary", "kind"} {+ if v, ok := b[key]; ok {+ out[key] = v+ }+ }+ out["source_doc_ids"] = unionStrs(metaStringSliceAny(a, "source_doc_ids"), metaStringSliceAny(b, "source_doc_ids"))+ out["source_chunk_ids"] = unionStrs(metaStringSliceAny(a, "source_chunk_ids"), metaStringSliceAny(b, "source_chunk_ids"))+ if v, ok := b["entity_names"]; ok {+ out["entity_names"] = v+ }+ if v, ok := b["related_kb_pages"]; ok {+ out["related_kb_pages"] = v+ }+ if v, ok := b["outlinks"]; ok {+ out["outlinks"] = v+ }+ return out+}++func unionStrs(a, b []string) []string {+ if len(a) == 0 {+ return b+ }+ if len(b) == 0 {+ return a+ }+ seen := make(map[string]struct{}, len(a)+len(b))+ out := make([]string, 0, len(a)+len(b))+ for _, s := range append(append([]string(nil), a...), b...) {+ if _, ok := seen[s]; ok {+ continue+ }+ seen[s] = struct{}{}+ out = append(out, s)+ }+ return out+}++// metaStringSliceAny reads a []string from a meta map that may box the value as+// []string or []any (engine/JSON round-trip does not guarantee a single type).+func metaStringSliceAny(m map[string]any, key string) []string {+ switch v := m[key].(type) {+ case []string:+ return v+ case []any:+ out := make([]string, 0, len(v))+ for _, e := range v {+ if s, ok := e.(string); ok {+ out = append(out, s)+ }+ }+ return out+ }+ return nil+}++// isWikiGroup reports whether a merge group targets the wiki variant and must use+// replace-only (never the JSON-merge decider).+func isWikiGroup(g MergeGroup) bool {+ return g.Existing.Variant == kccommon.VariantWiki+}++// wikiDecideBatch folds every wiki-group candidate into its existing row using+// replace-only semantics, WITHOUT any LLM call. It mutates the passed groups in+// place and returns them.+func wikiDecideBatch(_ context.Context, groups []MergeGroup) []MergeGroup {+ for gi := range groups {+ existing := groups[gi].Existing+ var distinct []kccommon.Product+ duplicated := false+ for _, cand := range groups[gi].Candidates {+ // A wiki page candidate is always a replacement of the existing row+ // (same slug, newer content). It is never kept as an additional distinct+ // row — distinctness for wiki is decided by KNN (different slug -> different+ // existing row -> different group).+ existing = wikiReplace(existing, cand)+ duplicated = true+ }+ groups[gi].Merged = existing+ groups[gi].Duplicate = duplicated+ groups[gi].Distinct = distinct+ }+ return groups+}diff --git a/internal/ingestion/knowledge_compile/wiki_merge_test.go b/internal/ingestion/knowledge_compile/wiki_merge_test.gonew file mode 100644index 0000000000..75ae2aa08c--- /dev/null+++ b/internal/ingestion/knowledge_compile/wiki_merge_test.go@@ -0,0 +1,89 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package knowledge_compile++import (+ "context"+ "testing"++ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// TestWikiReplaceCopiesCandidateVector locks the wiki replace-only merge contract+// (CodeRabbit Major): wikiReplace swaps the page body for the candidate's Markdown+// AND must copy the candidate's embedding too. WriteMerged persists Product.Vector+// without re-embedding, so keeping the existing row's vector would leave a stale+// embedding for the old page body and break KNN similarity searches.+func TestWikiReplaceCopiesCandidateVector(t *testing.T) {+ existing := kccommon.Product{+ ID: "kb/wiki/alpha",+ DocID: "kb",+ TenantID: "t1",+ Variant: kccommon.VariantWiki,+ Content: "# old markdown",+ Vector: []float32{1, 0, 0, 0},+ Meta: map[string]any{+ "kind": "page",+ "slug": "page/alpha",+ "created_at": "2024-01-01",+ },+ }+ candidate := kccommon.Product{+ ID: "kb/wiki/alpha",+ DocID: "kb",+ TenantID: "t1",+ Variant: kccommon.VariantWiki,+ Content: "# new markdown",+ Vector: []float32{0, 1, 0, 0},+ Meta: map[string]any{+ "kind": "page",+ "slug": "page/alpha",+ "run_id": "run-9",+ },+ }++ groups := wikiDecideBatch(context.Background(), []MergeGroup{{+ Existing: existing,+ Candidates: []kccommon.Product{candidate},+ }})++ merged := groups[0].Merged+ if merged.Content != "# new markdown" {+ t.Fatalf("content = %q, want candidate markdown", merged.Content)+ }+ if !groups[0].Duplicate {+ t.Fatalf("wiki candidate must be marked duplicate (replacement)")+ }+ if len(merged.Vector) != len(candidate.Vector) {+ t.Fatalf("merged vector must be copied from the candidate, got %v", merged.Vector)+ }+ for i := range candidate.Vector {+ if merged.Vector[i] != candidate.Vector[i] {+ t.Fatalf("merged vector[%d] = %v, want candidate %v", i, merged.Vector, candidate.Vector)+ }+ }+ // Identity and creation time come from the existing row; content-bearing and+ // page metadata fields come from the candidate.+ if merged.ID != existing.ID || merged.DocID != existing.DocID {+ t.Fatalf("identity must be preserved: id=%q doc_id=%q", merged.ID, merged.DocID)+ }+ if merged.Meta["created_at"] != "2024-01-01" {+ t.Fatalf("creation time must be preserved from the existing row, got %v", merged.Meta["created_at"])+ }+ if merged.Meta["run_id"] != "run-9" {+ t.Fatalf("run_id must come from the candidate, got %v", merged.Meta["run_id"])+ }+}diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 15a006cbbe..97d2450540 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -60,6 +60,14 @@ type Writer interface {ProjectWikiGraph(ctx context.Context, tenant, kb string) error// DropWikiGraph deletes every wiki_entity / wiki_relation row for the dataset.DropWikiGraph(ctx context.Context, tenant, kb string) error+ // DeleteMerged removes the dataset-level merged rows for a KB so an+ // incremental build can start from a clean slate. The structural filter+ // deletes only rows produced by the dataset-level merge — kb_id == kb AND+ // available_int == 1 AND compile_kwd is a wiki variant (wiki_page/+ // wiki_section). Per-document rows (doc_id == doc, available_int == 0) and+ // rows for other tenants / variants are untouched. The match_kwd guard is the+ // in-memory safety net; the structural filter is the source of truth.+ DeleteMerged(ctx context.Context, tenant, kb string) error}// engineWriter persists dataset-level merged products through the global@@ -188,6 +196,26 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm"create_time": now.Format("2006-01-02 15:04:05"),"create_timestamp_flt": float64(now.Unix()),}+ // wiki_incremental port: persist the product kind so the Reader can round-trip+ // page vs section without re-deriving it from compile_kwd. The merged writer+ // carries the authoritative kc_kind; legacy rows without it are derived in+ // productFromChunkMap (compile_kwd wiki_page -> "page", wiki_section ->+ // "section"). Without this, the dataset-level merge could not distinguish a+ // wiki page from a section and the processBatch "Meta.kind==page" filter would+ // be unreliable.+ if kind := metaString(p.Meta, "kind"); kind != "" {+ m["kc_kind"] = kind+ }+ // wiki_incremental port: preserve the original creation timestamp across a+ // replace-only merge. If the incoming merged product already carries+ // created_at_unix (restored by the Reader from create_timestamp_flt), reuse+ // it; otherwise stamp a fresh now() (first creation). This is what stops every+ // rebuild from re-stamping the creation time.+ if v, ok := metaFloat(p.Meta, "created_at_unix"); ok {+ m["create_timestamp_flt"] = v+ // Rebuild the human-readable form from the preserved unix time.+ m["create_time"] = time.Unix(int64(v), 0).Format("2006-01-02 15:04:05")+ }// Carry the wiki page metadata onto the merged row so the dataset-level// products keep the fields the artifact API (ListArtifacts/ListWikiTopics)// and page renderers read. Without this the merged rows lose page_type_kwd /@@ -454,6 +482,42 @@ func metaInt(m map[string]any, key string) (int64, bool) {return 0, false}+// metaFloat extracts a float64 from a map value that may be boxed as float64,+// int64, int, or string — the engine/JSON round-trip does not guarantee a+// single numeric type. Used to recover create_timestamp_flt so the reader can+// preserve the original creation time across a replace-only merge.+func metaFloat(m map[string]any, key string) (float64, bool) {+ switch v := m[key].(type) {+ case float64:+ return v, true+ case int64:+ return float64(v), true+ case int:+ return float64(v), true+ case string:+ var f float64+ if _, err := fmt.Sscanf(v, "%f", &f); err == nil {+ return f, true+ }+ }+ return 0, false+}++// kwdToVariant is the inverse of compileKwdForVariant: it maps a stored+// compile_kwd back to its compiler Variant. Both wiki_page and wiki_section+// map to VariantWiki (same product family); the page/section distinction is+// carried by the kc_kind field, not the variant. Returns an error for an+// unknown kwd so callers can reject dirty/foreign rows.+func kwdToVariant(kwd string) (kccommon.Variant, error) {+ switch kwd {+ case compileKwdWikiPage, compileKwdWikiSection, compileKwdWikiEntity, compileKwdWikiRelation:+ return kccommon.VariantWiki, nil+ case string(kccommon.VariantStructure), string(kccommon.VariantTree), string(kccommon.VariantMindmap):+ return kccommon.Variant(kwd), nil+ }+ return "", fmt.Errorf("unknown compile_kwd %q", kwd)+}+// --- Wiki page graph materialization (wiki_entity / wiki_relation) ---// compile_kwd values for the dataset-level products this package writes. The@@ -463,6 +527,7 @@ func metaInt(m map[string]any, key string) (int64, bool) {// dedicated wiki_entity / wiki_relation buckets.const (compileKwdWikiPage = "wiki_page"+ compileKwdWikiSection = "wiki_section"compileKwdWikiEntity = "wiki_entity"compileKwdWikiRelation = "wiki_relation"compileKwdWikiPageGraph = "wiki_page_graph" // legacy Python blob, swept on drop@@ -569,6 +634,34 @@ func (w engineWriter) DropWikiGraph(ctx context.Context, tenant, kb string) erroreturn w.dropWikiGraph(ctx, tenant, kb)}+// DeleteMerged removes the dataset-level (available_int=1) wiki merged rows for+// a KB so an incremental build can start from a clean slate. The structural+// filter (kb_id + available_int=1 + wiki page/section compile_kwd variants) is+// the source of truth; it never targets per-document rows (available_int=0) nor+// rows of other tenants / variants, so a wrong tenantID / kb cannot cascade.+func (w engineWriter) DeleteMerged(ctx context.Context, tenant, kb string) error {+ eng := w.eng+ if eng == nil {+ eng = engine.Get()+ }+ if eng == nil {+ return nil+ }+ baseName := fmt.Sprintf("ragflow_%s", tenant)+ _, err := eng.DeleteChunks(ctx, map[string]interface{}{+ "kb_id": kb,+ "available_int": 1,+ "compile_kwd": []string{+ compileKwdWikiPage,+ compileKwdWikiSection,+ },+ }, baseName, kb)+ if err != nil {+ return fmt.Errorf("delete merged: %w", err)+ }+ return nil+}+// dropWikiGraph is the shared delete path for both ProjectWikiGraph (when the// page set is empty, or before re-inserting) and DropWikiGraph. It deletes by// kb_id + compile_kwd IN (the graph buckets), also sweeping any legacy@@ -773,9 +866,6 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,// bare slug (the latter is what the LLM emits in wikitext links); resolve// both so a bare outlink still produces an edge.for _, tgt := range p.Outlinks {- if tgt == p.Slug {- continue // self-loop: Python skips src == tgt- }tp, ok := bySlug[tgt]if !ok {// Bare-slug outlink: normalize (strip prefix, "_"->"-") and look@@ -793,6 +883,9 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,if !ok {continue // dangling edge: target page not in this projection}+ if tgt == p.Slug {+ continue // self-loop: Python skips src == tgt (full or bare slug)+ }relID := wikiGraphXXHash("wiki_relation", kb, p.Slug+":"+tgt)if seen[relID] {continuediff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.goindex b45a04f5c1..9d445b2f89 100644--- a/internal/ingestion/knowledge_compile/writer_test.go+++ b/internal/ingestion/knowledge_compile/writer_test.go@@ -1,6 +1,7 @@package knowledge_compileimport (+ "context""testing""time"@@ -87,7 +88,7 @@ func TestProductFromChunkMapRestoresWikiFields(t *testing.T) {"related_kb_pages_kwd": []interface{}{"entity/beta"},"section_level_int": float64(2),}- p, ok := productFromChunkMap(c, "t1")+ p, ok := productFromChunkMap(c, "t1", kccommon.VariantWiki)if !ok {t.Fatalf("productFromChunkMap returned not-ok")}@@ -110,3 +111,73 @@ func TestProductFromChunkMapRestoresWikiFields(t *testing.T) {t.Errorf("per-doc row must not be marked merged")}}++// TestDeleteMergedScopesToMergedWikiRows locks the W1 contract: DeleteMerged+// must only target dataset-level (available_int=1) wiki merged rows. The+// structural filter (kb_id + available_int + wiki page/section compile_kwd) is+// the source of truth, so a wrong tenant/kb can never cascade to per-document+// rows or to rows of other variants.+func TestDeleteMergedScopesToMergedWikiRows(t *testing.T) {+ eng := &fakeEngine{}+ w := engineWriter{eng: eng}+ if err := w.DeleteMerged(context.Background(), "t1", "kb1"); err != nil {+ t.Fatalf("DeleteMerged: %v", err)+ }+ cond := eng.lastDeleteCond+ if cond == nil {+ t.Fatal("engine.DeleteChunks was not called")+ }+ if cond["kb_id"] != "kb1" {+ t.Errorf("DeleteMerged kb_id = %v, want kb1", cond["kb_id"])+ }+ if cond["available_int"] != 1 {+ t.Errorf("DeleteMerged must scope to available_int=1, got %v", cond["available_int"])+ }+ variants, ok := cond["compile_kwd"].([]string)+ if !ok {+ t.Fatalf("DeleteMerged must pass a compile_kwd string slice, got %T", cond["compile_kwd"])+ }+ if len(variants) != 2 || variants[0] != compileKwdWikiPage || variants[1] != compileKwdWikiSection {+ t.Errorf("DeleteMerged compile_kwd = %v, want [%q %q]", variants, compileKwdWikiPage, compileKwdWikiSection)+ }+}++// TestProductFromChunkMapRejectsDirtyKwd locks the dirty-row contract from the+// wiki_incremental plan (Claim 4): productFromChunkMap must reverse-map the raw+// compile_kwd via kwdToVariant and reject any row whose kwd does not map to the+// expected variant, including unknown / malformed kinds (e.g. "artifact_page",+// "garbage", empty). It must not silently fall back to a raw-string comparison.+func TestProductFromChunkMapRejectsDirtyKwd(t *testing.T) {+ base := map[string]interface{}{+ "id": "wiki/1",+ "doc_id": "d1",+ "content_with_weight": "# Alpha",+ }+ dirtyKwds := []string{"artifact_page", "garbage", ""}+ for _, kwd := range dirtyKwds {+ c := map[string]interface{}{}+ for k, v := range base {+ c[k] = v+ }+ c["compile_kwd"] = kwd+ if p, ok := productFromChunkMap(c, "t1", kccommon.VariantWiki); ok {+ t.Errorf("dirty compile_kwd %q should be rejected, got product %+v", kwd, p)+ }+ }++ // Note: wiki_section maps to VariantWiki (page and section share the wiki+ // variant); the page/section distinction lives in Meta.kind and is enforced+ // by filterWikiPageCandidates, NOT by the variant dirty-row check. So a+ // wiki_section row legitimately satisfies a VariantWiki query here — the+ // section is dropped later by the kind filter.++ // A clean wiki_page row must pass for VariantWiki.+ good := map[string]interface{}{}+ for k, v := range base {+ good[k] = v+ }+ good["compile_kwd"] = compileKwdWikiPage+ if _, ok := productFromChunkMap(good, "t1", kccommon.VariantWiki); !ok {+ t.Errorf("clean wiki_page row should satisfy VariantWiki query")+ }+}diff --git a/internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.go b/internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.goindex 7a23c9e57f..b0208af05d 100644--- a/internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.go+++ b/internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.go@@ -242,6 +242,33 @@ func TestKnowledgeCompilerDSL_ParamBinding(t *testing.T) {if p.EnableHistoricalDedup {t.Errorf("EnableHistoricalDedup default = true, want false")}+ if p.Plan != nil {+ t.Errorf("Plan = %v, want nil when omitted from DSL", *p.Plan)+ }++ for _, tc := range []struct {+ name string+ plan bool+ }{+ {name: "mode_a", plan: false},+ {name: "mode_b", plan: true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ parsed, err := kc.ParseParam(map[string]any{+ "compilation_template_id": "t1",+ "plan": tc.plan,+ })+ if err != nil {+ t.Fatalf("ParseParam: %v", err)+ }+ if parsed.Plan == nil || *parsed.Plan != tc.plan {+ t.Fatalf("Plan = %v, want explicit %t", parsed.Plan, tc.plan)+ }+ if parsed.PlanEnabled() != tc.plan {+ t.Errorf("PlanEnabled() = %t, want %t", parsed.PlanEnabled(), tc.plan)+ }+ })+ }}// TestKnowledgeCompilerDSL_KindToVariant locks the compilation_template.kind ->diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json b/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.jsonindex db0ab19d38..dd2f890d62 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json@@ -187,9 +187,9 @@"obj": {"component_name": "Compiler","params": {+ "compilation_template_group_id": "","llm_id": "",- "variant": "structure",- "language": "English"+ "plan": false}},"upstream": [diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex fc483af648..0f2268c4b7 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -51,7 +51,7 @@ var builtinComponentParamsGolden = map[string]string{"qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}","resume": "{\"Extractor:ThreeDrinksAct\": {\"field_name\": \"metadata\", \"frequencyPenaltyEnabled\": true, \"frequency_penalty\": 0.7, \"llm_id\": \"THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW\", \"maxTokensEnabled\": false, \"max_tokens\": 256, \"presencePenaltyEnabled\": true, \"presence_penalty\": 0.4, \"prompts\": [{\"content\": \"Content: {TitleChunker:FlatMiceFix@chunks}\", \"role\": \"user\"}], \"sys_prompt\": \"Act as a precise resume metadata extractor. Extract stable, chunk-supported metadata from the provided resume content.\\n\\nRules:\\n1. Use only information explicitly stated in the content. Do not infer, guess, normalize, or add missing facts.\\n2. The input may be only one chunk of a resume. Extract only what this content directly supports.\\n3. Use only these field names:\\ncandidate_name, gender, phone, email, city, location, nationality, linkedin, github, website, highest_degree, degree_levels, school_names, majors, graduation_years, work_experience_years, current_job_title, job_titles, company_names, job_experience, industries, target_job_titles, target_locations, employment_types, skills, certificates, awards, summary_tags\\n4. Ignore detailed responsibilities, project descriptions, achievement narratives, self-evaluation, and other low-value local details.\\n5. Keep values in the same language as the source text whenever possible.\\n6. Remove duplicates and keep only concise, high-value metadata.\\n7. Return only fields that are explicitly supported by the content. Do not return empty or unsupported fields.\\n\\nField guidance:\\n- highest_degree: highest explicit degree level mentioned\\n- degree_levels: all explicit degree levels mentioned\\n- school_names: explicit school, college, or university names\\n- majors: explicit fields of study\\n- graduation_years: explicit graduation years only\\n- work_experience_years: only if explicitly stated\\n- current_job_title: only if explicitly current or most recent\\n- job_titles: explicit role titles\\n- company_names: explicit employer names\\n- job_experience: concise structured work entries explicitly supported by the content, preferably including title, company, and time information when available\\n- industries: explicit industry names only\\n- target_job_titles: explicit desired roles only\\n- target_locations: explicit desired work locations only\\n- skills: concise, core, search-useful skills explicitly mentioned\\n- certificates: explicit certificate names only\\n- awards: explicit award names only\\n- summary_tags: short, high-value tags strictly supported by the content\\n\\nReturn only the extracted metadata. Do not output explanatory text.\", \"temperature\": 0.1, \"temperatureEnabled\": true, \"tenant_llm_id\": 29, \"topPEnabled\": true, \"top_p\": 0.3, \"auto_keywords\": 0, \"auto_questions\": 0, \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}, \"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": true, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": true, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:FlatMiceFix\": {\"hierarchy\": 1, \"include_heading_content\": false, \"levels\": [[\"^\\\\s*(?i:(?:\\\\d+[\\\\.\\\\)]\\\\s*)?(?:EDUCATION|ACADEMIC\\\\s*BACKGROUND|ACADEMIC\\\\s*HISTORY|EDUCATIONAL\\\\s*BACKGROUND|RELEVANT\\\\s*COURSEWORK|COURSEWORK|EXPERIENCE|WORK\\\\s*EXPERIENCE|PROFESSIONAL\\\\s*EXPERIENCE|RELEVANT\\\\s*EXPERIENCE|EMPLOYMENT\\\\s*HISTORY|CAREER\\\\s*HISTORY|INTERNSHIP\\\\s*EXPERIENCE|PROJECTS|PROJECT\\\\s*EXPERIENCE|ACADEMIC\\\\s*PROJECTS|PROFESSIONAL\\\\s*PROJECTS|SKILLS|TECHNICAL\\\\s*SKILLS|CORE\\\\s*COMPETENCIES|COMPETENCIES|QUALIFICATIONS|SUMMARY\\\\s*OF\\\\s*QUALIFICATIONS|CERTIFICATIONS|LICENSES|CERTIFICATES|AWARDS|HONORS|HONOURS|ACHIEVEMENTS|PUBLICATIONS|RESEARCH|RESEARCH\\\\s*EXPERIENCE|LEADERSHIP|LEADERSHIP\\\\s*EXPERIENCE|ACTIVITIES|EXTRACURRICULAR\\\\s*ACTIVITIES|ACTIVITIES\\\\s*(?:&|AND)\\\\s*SKILLS|INVOLVEMENT|CAMPUS\\\\s*INVOLVEMENT|VOLUNTEER\\\\s*EXPERIENCE|VOLUNTEERING|COMMUNITY\\\\s*SERVICE|LANGUAGES|INTERESTS|HOBBIES|PROFILE|PROFESSIONAL\\\\s*PROFILE|SUMMARY|PROFESSIONAL\\\\s*SUMMARY|CAREER\\\\s*SUMMARY|OBJECTIVE|CAREER\\\\s*OBJECTIVE|PERSONAL\\\\s*INFORMATION|CONTACT\\\\s*INFORMATION|ADDITIONAL\\\\s*INFORMATION|TRAINING))\\\\s*[::]?\\\\s*$\"], [\"^\\\\s*(?:\\\\d+[\\\\.、\\\\)]\\\\s*)?(?:教育背景|教育经历|学历背景|学术背景|技术背景|工作经历|工作经验|实习经历|项目经历|项目经验|科研经历|研究经历|校园经历|实践经历|专业经历|职业经历|技能|专业技能|技能特长|核心技能|技术栈|个人技能|工作技能|职业技能|技能与评价|技能与自我评价|工作技能与自我评价|职业技能与自我评价|证书|资格证书|职业资格|资质证书|获奖情况|获奖经历|荣誉|荣誉奖项|奖项|科研成果|论文发表|发表论文|领导经历|学生工作|校园活动|社团经历|活动经历|志愿经历|志愿服务|社会实践|语言能力|语言|自我评价|个人评价|自我总结|个人总结|个人优势|个人简介|个人信息|基本信息|联系方式|求职意向|应聘意向|职业目标|求职目标|兴趣爱好|兴趣特长|培训经历|其他信息|附加信息)\\\\s*[::]?\\\\s*$\"]], \"method\": \"hierarchy\"}, \"Tokenizer:KindHandsWin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}","table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",- "knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"language\": \"English\", \"llm_id\": \"\", \"variant\": \"structure\"}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",+ "knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"compilation_template_group_id\": \"\", \"llm_id\": \"\", \"plan\": false}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",}// Per-template test methods. Each resolves default component params from adiff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex 29eafe2f96..8907f31ba8 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -1912,7 +1912,9 @@ Example: Virtual Hosted Style`,instruction: 'Instruction',globalRules: 'Global rules',globalRulesPlaceholder: 'Input global compilation rules',- plan: 'Plan (grouping wiki pages by topic via LLM)',+ plan: 'Plan',+ planTip:+ 'Off: one wiki page per entity/concept. On: let the LLM decide to combine some entities/concepts into a single wiki page.',raptorTreeSettings: 'RAPTOR tree settings',summarizationPrompt: 'Summarization prompt',maxToken: 'Max token',diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex 22e7eb4839..6ecf82678e 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -1599,7 +1599,8 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系instruction: 'Instruction',globalRules: '全局规则',globalRulesPlaceholder: '请输入全局编译规则',- plan: 'Plan (LLM 分组合并 wiki 页面)',+ plan: 'Plan',+ planTip: '关闭:每个实体或概念对应一个 wiki 页面。开启:让 LLM 决定将某些实体/概念组合到单个 wiki 页面。',raptorTreeSettings: 'RAPTOR 树设置',summarizationPrompt: '摘要提示词',maxToken: '最大 token 数',diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsxindex 3b26171529..78032c1c2d 100644--- a/web/src/pages/agent/constant/pipeline.tsx+++ b/web/src/pages/agent/constant/pipeline.tsx@@ -363,6 +363,7 @@ export const initialExtractorValues = {export const initialCompilationValues = {compilation_template_group_id: '',llm_id: '',+ plan: false,outputs: {chunks: { type: 'Array<Object>', value: [] },},diff --git a/web/src/pages/agent/form/compilation-form/index.tsx b/web/src/pages/agent/form/compilation-form/index.tsxindex 1cdbf0fc02..5b1dd00558 100644--- a/web/src/pages/agent/form/compilation-form/index.tsx+++ b/web/src/pages/agent/form/compilation-form/index.tsx@@ -1,6 +1,8 @@import { CompilationTemplateFormField } from '@/components/compilation-template-form-field';import { LargeModelFormField } from '@/components/large-model-form-field';+import { SwitchFormField } from '@/components/switch-fom-field';import { Form } from '@/components/ui/form';+import { useTranslate } from '@/hooks/common-hooks';import { zodResolver } from '@hookform/resolvers/zod';import { memo } from 'react';import { useForm } from 'react-hook-form';@@ -18,6 +20,7 @@ import { Output } from '../components/output';export const FormSchema = z.object({compilation_template_group_id: z.string().optional(),llm_id: z.string().optional(),+ plan: z.boolean(),});export type CompilationFormSchemaType = z.infer<typeof FormSchema>;@@ -31,6 +34,7 @@ const CompilationForm = ({}: INextOperatorForm) => {const defaultValues = useFormValues(initialCompilationValues, node);const ownerTenantId = useOwnerTenantId();+ const { t } = useTranslate('setting');const form = useForm<CompilationFormSchemaType>({defaultValues,@@ -49,6 +53,11 @@ const CompilationForm = ({name="llm_id"ownerTenantId={ownerTenantId}></LargeModelFormField>+ <SwitchFormField+ name="plan"+ label={t('plan')}+ tooltip={t('planTip')}+ ></SwitchFormField></FormWrapper>{!hideOutputs && (<div className="p-5">