RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Port dataset-level knowledge compile to Go with variant dispatch (#18161)
Zhichang Yuc677e9af365f
diff --git a/internal/ingestion/component/knowledge_compiler/common/types_test.go b/internal/ingestion/component/knowledge_compiler/common/types_test.gonew file mode 100644index 0000000000..38e3da1742--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/common/types_test.go@@ -0,0 +1,53 @@+package common++import "testing"++// TestKindToVariant_WhitelistCoversAllValidKinds covers O2a: every whitelisted+// template kind maps to a non-empty variant without error.+func TestKindToVariant_WhitelistCoversAllValidKinds(t *testing.T) {+ validKinds := []string{+ "tree", "mind_map", "mindmap", "wiki", "page_index",+ "session_essence", "session_graph", "timeline",+ "knowledge_graph", "structure", "knowledgegraph", "graph",+ }+ for _, kind := range validKinds {+ v, err := KindToVariant(kind)+ if err != nil {+ t.Errorf("KindToVariant(%q) unexpected error: %v", kind, err)+ }+ if v == "" {+ t.Errorf("KindToVariant(%q) returned empty variant", kind)+ }+ }+}++// TestKindToVariant_UnknownKindHardFails covers O2a: a kind outside the+// whitelist returns an error (no fallback) so the rebuild/component path cannot+// silently mis-route an unknown template.+func TestKindToVariant_UnknownKindHardFails(t *testing.T) {+ for _, kind := range []string{"garbage", "artifact_page", "unknown_xyz", ""} {+ if _, err := KindToVariant(kind); err == nil {+ t.Errorf("KindToVariant(%q) should hard-fail (O2a), got nil error", kind)+ }+ }+}++// TestKindToVariant_CanonicalForms covers O2a aliases: the same template kind+// in different canonical spellings maps to the same variant.+func TestKindToVariant_CanonicalForms(t *testing.T) {+ a, err := KindToVariant("structure")+ if err != nil {+ t.Fatalf("structure: %v", err)+ }+ b, err := KindToVariant("knowledge_graph")+ if err != nil {+ t.Fatalf("knowledge_graph: %v", err)+ }+ c, err := KindToVariant("graph")+ if err != nil {+ t.Fatalf("graph: %v", err)+ }+ if a != b || a != c {+ t.Errorf("structure/knowledge_graph/graph should map to the same variant, got %q %q %q", a, b, c)+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex a52135dea8..aa0b23357b 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -6,7 +6,6 @@ package knowledge_compilerimport ("context"- "encoding/json""fmt""log""sort"@@ -21,7 +20,6 @@ import ("ragflow/internal/ingestion/component/knowledge_compiler/tree""ragflow/internal/ingestion/component/knowledge_compiler/wiki""ragflow/internal/ingestion/component/schema"- "ragflow/internal/service/nav""ragflow/internal/tokenizer""go.uber.org/zap"@@ -135,9 +133,6 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in// template's id and kind. All products are buffered (no streaming sink), so// the post-run loop below covers every row (M1).var out common.Outputs- // navByProducts accumulates every tree/structure by-product job so each is- // written after the spec loop (not just the last one).- var navByProducts []navByProductfor _, spec := range specs {variant, err := common.KindToVariant(spec.Kind)if err != nil {@@ -185,13 +180,6 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inreturn nil, err}- // Accumulate tree/structure by-product jobs so each is written after the- // spec loop (a multi-spec batch must not drop any spec's products). nav is- // a derived artifact; its failures are logged and never abort the pipeline.- if variant == common.VariantTree || variant == common.VariantStructure {- navByProducts = append(navByProducts, navByProduct{deps: deps, variant: variant, products: o.Products})- }-for i := range o.Products {if o.Products[i].Meta == nil {o.Products[i].Meta = map[string]any{}@@ -204,17 +192,6 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inout.Products = append(out.Products, o.Products...)}- // Write each dataset-nav by-product after all specs ran. tree by-product- // (Python compiler.py:475) and structure/page_index by-product (Python- // runner.py:189). Failures are logged and never abort the pipeline.- for _, job := range navByProducts {- if job.variant == common.VariantTree {- upsertTreeNav(ctx, job.deps, in.DocID, job.products)- } else {- upsertStructureNav(ctx, job.deps, in.DocID, job.products)- }- }-// Convert the compiled products into chunk-aligned docs (matching// conf/infinity_mapping.json) and merge them into the upstream input// chunks. The component stays DB-independent and no longer routes through a@@ -252,142 +229,6 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inreturn mergeChunks(inputs, compiled), nil}-// navByProduct is one deferred dataset-nav by-product job: the compile variant-// that produced it and the products to summarize from.-type navByProduct struct {- deps common.Deps- variant common.Variant- products []common.Product-}--// upsertTreeNav writes the dataset-nav by-product after tree compilation. It-// finds the tree root product (Meta kind=root) whose Content is the document-// summary, and places it into the ES-backed nav tree. Any failure (missing nav-// service, missing root, embedding/upsert error) is logged and skipped — the-// nav artifact must never block the compile pipeline.-func upsertTreeNav(ctx context.Context, deps common.Deps, docID string, products []common.Product) {- ns := nav.GetNavService()- if ns == nil {- log.Printf("knowledge_compiler: datasetnav by-product skipped (NavService not initialized)")- return- }- var summary string- var vec []float32- for i := range products {- if kind, _ := products[i].Meta["kind"].(string); kind == "root" {- summary = products[i].Content- vec = products[i].Vector- break- }- }- if strings.TrimSpace(summary) == "" {- log.Printf("knowledge_compiler: datasetnav by-product skipped (tree has no root summary)")- return- }- if len(vec) == 0 {- if deps.Embed == nil {- log.Printf("knowledge_compiler: datasetnav by-product skipped (no embedder, no precomputed vector)")- return- }- embeddings, err := deps.Embed.Encode(ctx, []string{summary})- if err != nil {- log.Printf("knowledge_compiler: datasetnav by-product skipped (embedding failed): %v", err)- return- }- if len(embeddings) == 0 {- log.Printf("knowledge_compiler: datasetnav by-product skipped (embedding produced no vector)")- return- }- vec = embeddings[0]- }- if err := ns.UpsertDoc(ctx, nav.UpsertDocInput{- TenantID: deps.TenantID,- KbID: deps.DatasetID,- DocID: docID,- Summary: summary,- Embedd: vec,- }); err != nil {- // Log and continue: nav is a derived artifact, never abort compile.- log.Printf("knowledge_compiler: datasetnav by-product upsert failed (continuing): %v", err)- }-}--// upsertStructureNav writes the dataset-nav by-product after structure/page_index-// compilation (mirroring Python runner.py:189 _upsert_dataset_nav_from_page_index).-// It finds the graph product (Meta kind=graph), summarizes its entity-// descriptions into a document-level summary, and places it into the nav tree.-// Failures are logged and never abort the compile pipeline.-func upsertStructureNav(ctx context.Context, deps common.Deps, docID string, products []common.Product) {- ns := nav.GetNavService()- if ns == nil {- log.Printf("knowledge_compiler: datasetnav by-product skipped (NavService not initialized)")- return- }- var graph *common.Product- for i := range products {- if kind, _ := products[i].Meta["kind"].(string); kind == "graph" {- graph = &products[i]- break- }- }- if graph == nil {- return- }- summary := pageIndexSummary(graph.Content)- if strings.TrimSpace(summary) == "" {- log.Printf("knowledge_compiler: datasetnav by-product skipped (structure graph has no entity descriptions)")- return- }- // Embed the SUMMARY text, not the graph JSON — the nav doc's vector must- // represent the summary semantics for the KNN router to match correctly.- if deps.Embed == nil {- log.Printf("knowledge_compiler: datasetnav by-product skipped (no embedder)")- return- }- embeddings, err := deps.Embed.Encode(ctx, []string{summary})- if err != nil || len(embeddings) == 0 {- log.Printf("knowledge_compiler: datasetnav by-product skipped (structure embedding failed): %v", err)- return- }- vec := embeddings[0]- if err := ns.UpsertDoc(ctx, nav.UpsertDocInput{- TenantID: deps.TenantID,- KbID: deps.DatasetID,- DocID: docID,- Summary: summary,- Embedd: vec,- }); err != nil {- log.Printf("knowledge_compiler: datasetnav by-product upsert failed (continuing): %v", err)- }-}--// pageIndexSummary concatenates entity descriptions from a structure graph JSON-// ({"entities": [{"name","description"}, ...]}), producing a document-level-// summary for dataset navigation.-func pageIndexSummary(graphJSON string) string {- var graph struct {- Entities []struct {- Name string `json:"name"`- Description string `json:"description"`- } `json:"entities"`- }- if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {- return ""- }- var b strings.Builder- for _, e := range graph.Entities {- desc := strings.Join(strings.Fields(e.Description), " ")- if desc == "" {- continue- }- if e.Name != "" {- b.WriteString(e.Name + ": ")- }- b.WriteString(desc + "\n")- }- return b.String()-}-// resolveTemplateSpecs resolves the configured compilation template spec(s) to// their TemplateInfo rows. Priority: compilation_template_id >// compilation_template_group_id. The group path resolves the group to its@@ -547,6 +388,14 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {if err := doc.SetExtraValue("compilation_template_kind_kwd", kindOrVariant(p)); err != nil {return nil, err}+ // scope_kwd marks doc/dataset-level rows (B8/O1=B). A doc-level compiled+ // product is always a per-document (scope="doc") input to the dataset-level+ // merge; the consumer rewrites dataset-level rows with scope_kwd="dataset".+ // This is the dataset-level writer-layer discriminator, not a doc-level+ // compile-internal concern.+ if err := doc.SetExtraValue("scope_kwd", "doc"); err != nil {+ return nil, err+ }if p.ParentID != "" {if err := doc.SetExtraValue("parent_kwd", p.ParentID); err != nil {return nil, errdiff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex d9f796e81a..dc055d1bcd 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -15,7 +15,6 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/knowledge_compiler/common"- "ragflow/internal/service/nav""gorm.io/gorm")@@ -258,124 +257,6 @@ func TestKnowledgeCompiler_Datasetnav_NoVariant(t *testing.T) {}}-// fakeNavSvc records the most recent UpsertDoc call so a test can assert the-// tree by-product hook feeds the nav service the root document summary.-type fakeNavSvc struct {- mu sync.Mutex- called bool- summary string- docID string- kbID string-}--func (f *fakeNavSvc) UpsertDoc(_ context.Context, in nav.UpsertDocInput) error {- f.mu.Lock()- defer f.mu.Unlock()- f.called = true- f.summary = in.Summary- f.docID = in.DocID- f.kbID = in.KbID- return nil-}-func (f *fakeNavSvc) RemoveDoc(context.Context, string, string, string) error { return nil }-func (f *fakeNavSvc) Search(context.Context, string, string, string, []float32, int) ([]nav.NavHit, error) {- return nil, nil-}-func (f *fakeNavSvc) ListClusters(context.Context, string, string, int, int) ([]nav.NavNode, int64, error) {- return nil, 0, nil-}-func (f *fakeNavSvc) ListChildren(context.Context, string, string, string, int, int) ([]nav.NavNode, int64, error) {- return nil, 0, nil-}--// TestKnowledgeCompiler_TreeNavByProduct asserts the tree compile-complete hook-// writes the dataset-nav by-product: after tree.Run produces a root product,-// upsertTreeNav calls NavService.UpsertDoc with the root summary and the-// doc/dataset context, and never aborts on failure.-func TestKnowledgeCompiler_TreeNavByProduct(t *testing.T) {- fake := &fakeNavSvc{}- nav.SetNavService(fake)- t.Cleanup(func() { nav.SetNavService(nil) })-- deps := common.Deps{- TenantID: "t1",- DatasetID: "kb1",- Chat: proseChat{},- Embed: mockEmbedder{dim: 8},- }- products := []common.Product{- {Content: "section one body", Meta: map[string]any{"kind": "summary", "level": 0}},- {Content: "overall doc theme root summary", Vector: []float32{0.1, 0.2}, Meta: map[string]any{"kind": "root", "level": -1}},- }- upsertTreeNav(context.Background(), deps, "d1", products)-- fake.mu.Lock()- defer fake.mu.Unlock()- if !fake.called {- t.Fatal("upsertTreeNav did not call NavService.UpsertDoc")- }- if fake.docID != "d1" {- t.Errorf("doc_id = %q, want d1", fake.docID)- }- if fake.kbID != "kb1" {- t.Errorf("kb_id = %q, want kb1", fake.kbID)- }- if !strings.Contains(fake.summary, "overall doc theme root summary") {- t.Errorf("summary = %q, want the tree root summary", fake.summary)- }-}--// TestKnowledgeCompiler_TreeNavByProduct_NilServiceDoesNotAbort asserts the-// by-product hook tolerates a missing NavService without erroring (nav is a-// derived artifact; its absence must never abort compilation).-func TestKnowledgeCompiler_TreeNavByProduct_NilServiceDoesNotAbort(t *testing.T) {- nav.SetNavService(nil)- t.Cleanup(func() { nav.SetNavService(nil) })- // Should not panic and must return normally.- upsertTreeNav(context.Background(), common.Deps{}, "d1",- []common.Product{{Content: "x", Meta: map[string]any{"kind": "root"}}})-}--// TestKnowledgeCompiler_StructureNavByProduct asserts the structure/page_index-// by-product hook summarizes the graph product entities and feeds NavService.-func TestKnowledgeCompiler_StructureNavByProduct(t *testing.T) {- fake := &fakeNavSvc{}- nav.SetNavService(fake)- t.Cleanup(func() { nav.SetNavService(nil) })-- graphJSON := `{"entities":[{"name":"Engine","description":"a propulsion device"},{"name":"Fuel","description":"combustion source"}]}`- products := []common.Product{- {Content: graphJSON, Vector: []float32{0.1, 0.2}, Meta: map[string]any{"kind": "graph", "compile_kwd": "page_index"}},- }- // The by-product embeds the SUMMARY (not the graph JSON vector), so an- // embedder must be present.- upsertStructureNav(context.Background(), common.Deps{TenantID: "t1", DatasetID: "kb1", Embed: mockEmbedder{dim: 8}}, "d1", products)-- fake.mu.Lock()- defer fake.mu.Unlock()- if !fake.called {- t.Fatal("upsertStructureNav did not call NavService.UpsertDoc")- }- if fake.docID != "d1" {- t.Errorf("doc_id = %q, want d1", fake.docID)- }- if !strings.Contains(fake.summary, "Engine: a propulsion device") {- t.Errorf("summary = %q, want entity descriptions", fake.summary)- }-}--// TestPageIndexSummary asserts entity descriptions are concatenated into a-// document summary.-func TestPageIndexSummary(t *testing.T) {- got := pageIndexSummary(`{"entities":[{"name":"A","description":"one thing"},{"name":"B","description":""}]}`)- if !strings.Contains(got, "A: one thing") {- t.Errorf("summary = %q, want entity A", got)- }- if strings.Contains(got, "B") {- t.Errorf("summary should skip empty-description entity, got %q", got)- }-}-func TestKnowledgeCompiler_Alias_Mindmap(t *testing.T) {installMockDeps(t)// "mind_map" is the deprecated alias for "mindmap"; both resolve to thediff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex 753b017201..dd1c0b5cd8 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -17,9 +17,11 @@ package knowledge_compileimport ("context"+ "encoding/json""errors""fmt""sort"+ "strings""sync""time"@@ -27,6 +29,7 @@ import ("ragflow/internal/dao""ragflow/internal/engine"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+ "ragflow/internal/service/nav""go.uber.org/zap""gorm.io/gorm"@@ -69,7 +72,7 @@ type Consumer struct {// satisfied by *mysqlScheduler (and FakeScheduler in tests). Using a narrow// interface keeps the Claimer surface unchanged.type rewriteScheduler interface {- Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error+ Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) errorCancelInflight(ctx context.Context, datasetID, token string) error}@@ -306,21 +309,37 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)// lock while the clear runs, so it can never repopulate cleared state with// old results.if err := c.scheduler.WithDatasetLock(ctx, kb, func(_ string) error {- if derr := c.writer.DeleteMerged(ctx, tenant, kb); derr != nil {+ // B4/B1b: a full rebuild clears EVERY variant the consumer manages+ // (wiki merged rows, structure scope_kwd="dataset" rows, and the nav+ // tree rows) via the fixed full-set clean (empty variants), not just the+ // wiki rows that DeleteMerged removes — otherwise removed-template ghost+ // rows survive. DropWikiGraph additionally sweeps the legacy wiki graph.+ if derr := c.writer.DeleteMergedForVariant(ctx, tenant, kb, nil); derr != nil {return derr}return c.writer.DropWikiGraph(ctx, tenant, kb)}); err != nil {- return fmt.Errorf("knowledge_compile: rebuild clear merged+graph: %w", err)+ return fmt.Errorf("knowledge_compile: rebuild clear all variants + graph: %w", err)}- // 4. enumerate every doc and republish.+ // 4. enumerate every doc and republish. Per B1, the republished events must+ // carry the doc's compile variants so the consumer can route to the correct+ // dataset-level path (a nil/empty Variants would fall back to the legacy+ // unified path and tree nav could not be rebuilt). The variants are recovered+ // from the doc-level compiled products' authoritative `compilation_template_kind_kwd`+ // (via KindToVariant, O2a whitelist hard-fail), not re-derived from `compile_kwd`.docs, err := c.docLister(ctx, tenant, kb)if err != nil {return fmt.Errorf("knowledge_compile: rebuild list docs: %w", err)}for _, docID := range docs {- if perr := rs.Publish(ctx, tenant, kb, docID, string(EventTypeCompleted)); perr != nil {+ variants, rerr := c.recoverDocVariants(ctx, tenant, kb, docID)+ if rerr != nil {+ // O2a hard failure: abort the rebuild so an unknown template kind+ // cannot leave the doc's dataset-level products unrebuilt.+ return fmt.Errorf("knowledge_compile: rebuild recover variants %s: %w", docID, rerr)+ }+ if perr := rs.Publish(ctx, tenant, kb, docID, string(EventTypeCompleted), variants); perr != nil {return fmt.Errorf("knowledge_compile: rebuild republish %s: %w", docID, perr)}}@@ -343,23 +362,77 @@ func defaultDocLister(ctx context.Context, tenant, kb string) ([]string, error)return dao.NewDocumentDAO().ListIDsByKBIDWithOptions(ctx, kcDB, dao.DocumentListOptions{KbID: kb})}-// filterWikiPageCandidates returns only the candidates the dataset-level merge-// is allowed to fold: for the wiki variant, that is strictly page-kind products-// (Meta.kind == "page"). Sections are a doc-level concern and must never enter-// the dataset-level page bucket, and the deduper must not carry an implicit-// section-filter contract. Non-wiki variants pass through unchanged. Legacy wiki-// rows whose kind is empty are derived in the Reader; only truly page-kind wiki-// products proceed.+// recoverDocVariants reconstructs the compile-type set a document produced, by+// reading its persisted doc-level products and mapping each product's+// authoritative `compilation_template_kind_kwd` through common.KindToVariant.+// It is used by RebuildDataset (B1) so the republished completed events carry+// the variants needed to route the dataset-level re-compile (tree nav, structure+// merge, wiki merge) — a nil/empty Variants would fall back to the legacy+// unified path and tree nav could not be rebuilt.+//+// KindToVariant (O2a) is a fixed whitelist that HARD-FAILS on unknown kinds: a+// product whose kind is not in the whitelist is a corrupt/unsupported state and+// the rebuild must abort (returning an error) rather than silently drop the+// variant and leave the doc's dataset-level products unrebuilt. A product+// without an authoritative kind falls back to its reverse-mapped variant (which+// itself must be whitelist-valid, enforced by KwdToVariant in the Reader). The+// returned slice is sorted and de-duplicated.+func (c *Consumer) recoverDocVariants(ctx context.Context, tenant, kb, docID string) ([]string, error) {+ products, err := c.reader.LoadDocProducts(ctx, tenant, kb, docID)+ if err != nil {+ return nil, fmt.Errorf("knowledge_compile: recover variants load doc %s: %w", docID, err)+ }+ seen := map[string]struct{}{}+ var out []string+ for _, p := range products {+ // Authoritative: product.Kind (compilation_template_kind_kwd) when present.+ if p.Kind != "" {+ v, err := kccommon.KindToVariant(p.Kind)+ if err != nil {+ // O2a hard failure: do not continue with an incomplete variant set.+ return nil, fmt.Errorf("knowledge_compile: recover variants doc %s kind %q: %w", docID, p.Kind, err)+ }+ if _, dup := seen[string(v)]; dup {+ continue+ }+ seen[string(v)] = struct{}{}+ out = append(out, string(v))+ continue+ }+ // Fallback: the product's reverse-mapped variant (already whitelist-valid).+ if p.Variant == "" {+ // No authoritative kind and no reverse-mapped variant: this is a+ // legacy/unknown row. Skip it so we never republish an empty variant+ // (which the consumer cannot route and defeats "nil = legacy").+ continue+ }+ if _, dup := seen[string(p.Variant)]; dup {+ continue+ }+ seen[string(p.Variant)] = struct{}{}+ out = append(out, string(p.Variant))+ }+ sort.Strings(out)+ return out, nil+}++// filterWikiPageCandidates returns ONLY the wiki page products the dataset-level+// wiki merge may fold. It drops every non-wiki product (tree/structure/nav) so+// those never enter the wiki Dedup/KNN/WriteMerged path, and within the wiki+// variant keeps strictly page-kind products (Meta.kind == "page"). Sections are+// a doc-level concern and must never enter the dataset-level page bucket, and+// the deduper must not carry an implicit section-filter contract. This is the+// per-variant product selection (review issue 2): a completed wiki-only doc whose+// event carried tree/structure products on a prior run must not feed stale+// tree/structure rows into the wiki merge.func filterWikiPageCandidates(candidates []kccommon.Product) []kccommon.Product {- // Allocate a fresh slice: reusing candidates[:0] would overwrite the caller's- // backing array in place, corrupting any other reference (e.g. the vector- // audit that still reads `incoming`).filtered := make([]kccommon.Product, 0, len(candidates))for _, cand := range candidates {- if cand.Variant == kccommon.VariantWiki {- if metaString(cand.Meta, "kind") != "page" {- continue- }+ if cand.Variant != kccommon.VariantWiki {+ continue // tree/structure products never enter the wiki merge+ }+ if metaString(cand.Meta, "kind") != "page" {+ continue}filtered = append(filtered, cand)}@@ -397,17 +470,19 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// (deleted last) semantics we want.type docState struct {winEvent EventType+ variants []string}byDoc := make(map[string]*docState, len(entries))for _, e := range entries {et := EventType(e.EventType)st := byDoc[e.DocID]if st == nil {- byDoc[e.DocID] = &docState{winEvent: et}+ byDoc[e.DocID] = &docState{winEvent: et, variants: e.Variants}continue}// Later entries overwrite earlier ones: last event wins.st.winEvent = et+ st.variants = e.Variants}for docID, st := range byDoc {switch st.winEvent {@@ -428,7 +503,9 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, eif _, hadTomb := tomb[docID]; hadTomb {pendingTombClear = append(pendingTombClear, docID)}- completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted)})+ // Preserve the winning event's variants so the dispatch below can+ // route this doc to the matching dataset-level compile path (A0-4).+ completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted), Variants: st.variants})}}// Sort for deterministic iteration in the delete/load passes below.@@ -518,6 +595,17 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e}return err}+ // G3: structure ghost cleanup — remove structure dataset rows whose only+ // source docs were just deleted. Also a locked destructive write.+ if err := c.withWriteLock(ctx, kb, token, func() error {+ return c.writer.DeleteStructureForDocs(ctx, tenant, kb, delIDs)+ }); err != nil {+ if errors.Is(err, errClaimSuperseded) {+ common.Info("knowledge_compile: batch stale before structure ghost cleanup, aborting (rewrite barrier)",+ zap.String("dataset_id", kb))+ }+ return err+ }}// --- Completion merge ---@@ -534,7 +622,44 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, eif err != nil {return err}- incoming = append(incoming, docProducts...)+ // A0-4: the event's Variants are the authoritative dispatch signal — route+ // ONLY the products whose variant the compiler actually produced for this+ // doc. A doc re-compiled under a new template may still hold stale+ // doc-level rows of an earlier variant in storage; without this gate those+ // would leak into the nav/structure paths despite the event asking only+ // for, say, wiki. Legacy events with empty Variants keep all products.+ incoming = append(incoming, productsForVariants(docProducts, e.Variants)...)+ }++ // A0-4 dispatch: route tree/structure products to the dataset-navigation tree+ // (NavService.UpsertDoc), and keep wiki (and only wiki) products on the+ // existing dataset-level merge path below. This is the per-variant dispatch+ // the plan requires: tree and structure both produce a nav by-product (B2),+ // so both upsert their summary into the cross-document nav tree; wiki products+ // continue to the replace-only merge.+ navIn := navInputFromProducts(kb, incoming)+ if len(navIn) > 0 {+ ns := nav.GetNavService()+ if ns == nil {+ common.Warn("knowledge_compile: nav service unavailable, skipping dataset-nav upsert for batch",+ zap.String("dataset_id", kb))+ } else if err := c.upsertNavLocked(ctx, tenant, kb, token, ns, navIn); err != nil {+ return err+ }+ }++ // G1+G2: dataset-level structure merge and build-time stamp, both under the+ // claim-fenced write lock (B7 applies to structure writes too): a worker+ // invalidated by CancelInflight must not write structure rows after a rebuild+ // has cleared them.+ if err := c.withWriteLock(ctx, kb, token, func() error {+ return c.mergeStructureDataset(ctx, tenant, kb, incoming)+ }); err != nil {+ if errors.Is(err, errClaimSuperseded) {+ common.Info("knowledge_compile: batch stale before structure merge, aborting (rewrite barrier)",+ zap.String("dataset_id", kb))+ }+ return err}// wiki_incremental port (M1): the dataset-level merge only processes wiki@@ -785,3 +910,248 @@ func candidateIdentity(p kccommon.Product) string {}return "<unknown>"}++// productsForVariants filters a doc's loaded products down to those whose+// variant is in the requested set. An empty set (legacy backlog entries without+// variants, or a rebuild event that carries none) is treated as "all variants"+// to preserve existing behavior; otherwise a product is kept only when its+// variant appears in variants. This is what makes BacklogEntry.Variants actually+// drive per-variant dispatch instead of handing every doc's full product set to+// every dataset-level path.+func productsForVariants(products []kccommon.Product, variants []string) []kccommon.Product {+ if len(variants) == 0 {+ return products+ }+ want := make(map[string]struct{}, len(variants))+ for _, v := range variants {+ want[v] = struct{}{}+ }+ out := products[:0:0]+ for i := range products {+ if _, ok := want[string(products[i].Variant)]; ok {+ out = append(out, products[i])+ }+ }+ return out+}++// navInputFromProducts extracts the dataset-navigation upsert inputs for the+// tree and structure products in a batch (B2: nav trigger is tree || structure).+// Summary extraction matches the component's by-product hooks:+// - tree: the root product (Meta.kind=="root") carries the document summary in+// Content and its vector in Vector.+// - structure: the graph product (Meta.kind=="graph") carries the graph JSON in+// Content; its entity descriptions are folded into a document-level summary+// via pageIndexSummary.+//+// Each document contributes at most one nav input, keyed by DocID, so a doc that+// yields several tree/structure products upserts once. NavService embeds the+// summary itself when Embedd is empty, so the consumer needs no embedder.+func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDocInput {+ type acc struct {+ in nav.UpsertDocInput+ }+ byDoc := make(map[string]*acc, len(products))+ for i := range products {+ p := &products[i]+ switch p.Variant {+ case kccommon.VariantTree:+ if kind, _ := p.Meta["kind"].(string); kind != "root" {+ continue+ }+ a := byDoc[p.DocID]+ if a == nil {+ byDoc[p.DocID] = &acc{in: nav.UpsertDocInput{TenantID: p.TenantID, KbID: kb, DocID: p.DocID}}+ a = byDoc[p.DocID]+ }+ a.in.Summary = p.Content+ a.in.Embedd = p.Vector+ case kccommon.VariantStructure:+ if kind, _ := p.Meta["kind"].(string); kind != "graph" {+ continue+ }+ a := byDoc[p.DocID]+ if a == nil {+ byDoc[p.DocID] = &acc{in: nav.UpsertDocInput{TenantID: p.TenantID, KbID: kb, DocID: p.DocID}}+ a = byDoc[p.DocID]+ }+ // Graph vector is NOT the summary vector; leave Embedd empty so+ // NavService embeds the folded summary text.+ a.in.Summary = pageIndexSummary(p.Content)+ }+ }+ out := make([]nav.UpsertDocInput, 0, len(byDoc))+ for _, a := range byDoc {+ if strings.TrimSpace(a.in.Summary) == "" {+ continue+ }+ out = append(out, a.in)+ }+ return out+}++// pageIndexSummary folds the entity descriptions of a structure graph JSON+// ({"entities":[{"name","description"},...]}) into a document-level summary for+// dataset navigation, matching the component's by-product logic.+func pageIndexSummary(graphJSON string) string {+ var graph struct {+ Entities []struct {+ Name string `json:"name"`+ Description string `json:"description"`+ } `json:"entities"`+ }+ if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {+ return ""+ }+ var b strings.Builder+ for _, e := range graph.Entities {+ desc := strings.Join(strings.Fields(e.Description), " ")+ if desc == "" {+ continue+ }+ if e.Name != "" {+ b.WriteString(e.Name)+ b.WriteString(": ")+ }+ b.WriteString(desc)+ b.WriteString("\n")+ }+ return b.String()+}++// mergeStructureDataset performs the dataset-level structure merge (G1/G4): it+// groups the structure products of a batch by (name, type), folds their+// descriptions and source doc/chunk sets into one StructureBucket per group, and+// writes scope_kwd="dataset" rows via WriteMergedStructure. This mirrors Python+// dataset_structure_merger._merge_bucket but is a self-contained from-scratch+// path (B3: the legacy unified merge never aggregated structure by name/type).+func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string, products []kccommon.Product) error {+ // Bucket by (lower(name), type) for entities and (lower(from), lower(to))+ // for relations — relations have no "name" and must not be silently dropped+ // (review issue 3).+ type ekey struct{ name, typ string }+ type rkey struct{ from, to string }+ entByKey := make(map[ekey]*StructureBucket, 16)+ relByKey := make(map[rkey]*StructureBucket, 16)+ for _, p := range products {+ if p.Variant != kccommon.VariantStructure {+ continue+ }+ kind := metaString(p.Meta, "kind")+ from := metaString(p.Meta, "from")+ to := metaString(p.Meta, "to")+ if kind == "relation" || (from != "" && to != "") {+ // Skip a relation with an incomplete endpoint pair: writing a bucket+ // like "A -> " would let multiple malformed relations collapse into+ // the same dataset row.+ if from == "" || to == "" {+ continue+ }+ k := rkey{from: strings.ToLower(from), to: strings.ToLower(to)}+ b := relByKey[k]+ if b == nil {+ b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to}+ relByKey[k] = b+ }+ appendBucket(b, p)+ continue+ }+ // entity bucket+ name := metaString(p.Meta, "name")+ typ := metaString(p.Meta, "entity_type")+ if typ == "" {+ typ = metaString(p.Meta, "type")+ }+ if name == "" {+ continue+ }+ k := ekey{name: strings.ToLower(name), typ: typ}+ b := entByKey[k]+ if b == nil {+ b = &StructureBucket{Name: name, Type: typ}+ entByKey[k] = b+ }+ appendBucket(b, p)+ }+ buckets := make([]StructureBucket, 0, len(entByKey)+len(relByKey))+ for _, b := range entByKey {+ buckets = append(buckets, *b)+ }+ for _, b := range relByKey {+ buckets = append(buckets, *b)+ }+ if len(buckets) == 0 {+ return nil+ }+ sort.Slice(buckets, func(i, j int) bool {+ if buckets[i].Name != buckets[j].Name {+ return buckets[i].Name < buckets[j].Name+ }+ return buckets[i].Type < buckets[j].Type+ })+ return c.writer.WriteMergedStructure(ctx, tenant, kb, buckets)+}++// appendBucket folds a structure product into a bucket: concatenates its+// description, unions its source doc/chunk ids, and folds its vector into a true+// element-wise running mean (review issue 13). VecCount tracks how many vectors+// have been folded so the mean is order-independent.+func appendBucket(b *StructureBucket, p kccommon.Product) {+ if strings.TrimSpace(p.Content) != "" {+ if b.Description != "" {+ b.Description += "\n"+ }+ b.Description += strings.TrimSpace(p.Content)+ }+ b.SourceDocIDs = appendUnique(b.SourceDocIDs, []string{p.DocID})+ b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, metaStringSlice(p.Meta, "source_chunk_ids"))+ if len(p.Vector) > 0 {+ b.VecCount+++ if len(b.Vector) == 0 {+ b.Vector = append([]float32(nil), p.Vector...)+ } else {+ n := len(p.Vector)+ if len(b.Vector) < n {+ b.Vector = append(b.Vector, make([]float32, n-len(b.Vector))...)+ }+ for i := 0; i < n; i++ {+ // running mean: acc + (next - acc)/count+ b.Vector[i] += (p.Vector[i] - b.Vector[i]) / float32(b.VecCount)+ }+ }+ }+}++// appendUnique appends only values not already present, preserving order.+func appendUnique(dst, values []string) []string {+ seen := make(map[string]bool, len(dst)+len(values))+ for _, v := range dst {+ seen[v] = true+ }+ for _, v := range values {+ if v == "" || seen[v] {+ continue+ }+ seen[v] = true+ dst = append(dst, v)+ }+ return dst+}++// upsertNavLocked places the batch's tree/structure summaries into the dataset+// navigation tree under the per-dataset write/rebuild lock, with the claim-token+// check inside the lock (B7: nav writes are claim-fenced like any other+// destructive write so a stale worker cannot repopulate a rebuilt tree).+func (c *Consumer) upsertNavLocked(ctx context.Context, tenant, kb, token string, ns nav.NavService, inputs []nav.UpsertDocInput) error {+ return c.withWriteLock(ctx, kb, token, func() error {+ for i := range inputs {+ // The minimal-loop NavService.UpsertDoc is idempotent by summary; a+ // failure here aborts the batch so the claim is not acked and the+ // batch is retried.+ if err := ns.UpsertDoc(ctx, inputs[i]); err != nil {+ return fmt.Errorf("knowledge_compile: nav upsert %s: %w", inputs[i].DocID, err)+ }+ }+ return nil+ })+}diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.goindex 9c4090615a..fa949389af 100644--- a/internal/ingestion/knowledge_compile/event.go+++ b/internal/ingestion/knowledge_compile/event.go@@ -103,21 +103,23 @@ func DefaultPublisher() Publisher { return defaultPublisher }func DefaultClaimer() Claimer { return defaultClaimer }// PublishCompleted records a doc_completed event: it appends the doc to the-// KB's durable MySQL backlog and wakes idle workers over NATS. It is a no-op-// when no Publisher has been installed (e.g. DB unavailable). A failure is+// KB's durable MySQL backlog and wakes idle workers over NATS. variants carries+// the doc-level compile types (tree/structure/wiki/mindmap) produced by this+// doc so the consumer can dispatch to the matching dataset-level path. It is a+// no-op when no Publisher has been installed (e.g. DB unavailable). A failure is// returned so callers can log but never fail the pipeline on it.-func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string) error {+func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {if defaultPublisher == nil {return nil}- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted))+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted), variants)}// PublishDeleted records a doc_deleted event the same way (Publish handles the-// append + notify pairing).+// append + notify pairing). Deleted events carry no compile types.func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string) error {if defaultPublisher == nil {return nil}- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted))+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), nil)}diff --git a/internal/ingestion/knowledge_compile/fake_engine_test.go b/internal/ingestion/knowledge_compile/fake_engine_test.goindex 48fe8e98ef..9e86549c09 100644--- a/internal/ingestion/knowledge_compile/fake_engine_test.go+++ b/internal/ingestion/knowledge_compile/fake_engine_test.go@@ -33,6 +33,10 @@ type fakeEngine struct {lastSearchReq *types.SearchRequest// lastDeleteCond captures the most recent DeleteChunks condition.lastDeleteCond map[string]interface{}+ // deleteConds accumulates every DeleteChunks condition in call order, so a+ // test can assert on a multi-delete clean (e.g. the full-set clean that+ // sweeps compile_kwd buckets and the scope_kwd="dataset" bucket separately).+ deleteConds []map[string]interface{}// deleteCount is the number returned by DeleteChunks.deleteCount int64}@@ -44,6 +48,7 @@ func (f *fakeEngine) Search(_ context.Context, req *types.SearchRequest) (*typesfunc (f *fakeEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, _, _ string) (int64, error) {f.lastDeleteCond = condition+ f.deleteConds = append(f.deleteConds, condition)return f.deleteCount, nil}diff --git a/internal/ingestion/knowledge_compile/nav_dispatch_test.go b/internal/ingestion/knowledge_compile/nav_dispatch_test.gonew file mode 100644index 0000000000..5fdd913fb4--- /dev/null+++ b/internal/ingestion/knowledge_compile/nav_dispatch_test.go@@ -0,0 +1,127 @@+package knowledge_compile++import (+ "reflect"+ "strings"+ "testing"++ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+ "ragflow/internal/service/nav"+)++// TestNavInputFromProducts_TreeAndStructure covers B2: nav is fed by BOTH tree+// and structure products (not tree alone). Tree uses the root product's summary;+// structure folds the graph entity descriptions into a summary via pageIndexSummary.+func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {+ products := []kccommon.Product{+ // tree: root product carries the doc summary + vector.+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,+ Content: "overall doc theme root summary", Vector: []float32{0.1, 0.2},+ Meta: map[string]any{"kind": "root"}},+ // tree: a non-root summary node is NOT a nav input.+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,+ Content: "section body", Meta: map[string]any{"kind": "summary", "level": 0}},+ // structure: graph product folds entity descriptions.+ {DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,+ Content: `{"entities":[{"name":"Engine","description":"a propulsion device"}]}`,+ Meta: map[string]any{"kind": "graph", "compile_kwd": "page_index"}},+ }++ got := navInputFromProducts("kb1", products)+ if len(got) != 2 {+ t.Fatalf("want 2 nav inputs (tree d1 + structure d2), got %d", len(got))+ }+ byDoc := map[string]nav.UpsertDocInput{}+ for _, in := range got {+ byDoc[in.DocID] = in+ }+ treeIn, ok := byDoc["d1"]+ if !ok {+ t.Fatal("missing tree nav input for d1")+ }+ if !strings.Contains(treeIn.Summary, "overall doc theme") {+ t.Errorf("tree summary = %q, want root summary", treeIn.Summary)+ }+ if !reflect.DeepEqual(treeIn.Embedd, []float32{0.1, 0.2}) {+ t.Errorf("tree embedd should be the root vector, got %v", treeIn.Embedd)+ }+ structIn, ok := byDoc["d2"]+ if !ok {+ t.Fatal("missing structure nav input for d2")+ }+ if !strings.Contains(structIn.Summary, "Engine: a propulsion device") {+ t.Errorf("structure summary = %q, want folded entity descriptions", structIn.Summary)+ }+ // Structure graph vector is NOT the summary vector: leave Embedd empty so+ // NavService embeds the folded summary text.+ if len(structIn.Embedd) != 0 {+ t.Errorf("structure embedd should be empty (NavService re-embeds), got %v", structIn.Embedd)+ }+}++// TestNavInputFromProducts_WikiExcluded covers the dispatch boundary: wiki+// products never become nav inputs (they go to the replace-only wiki merge).+func TestNavInputFromProducts_WikiExcluded(t *testing.T) {+ products := []kccommon.Product{+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantWiki,+ Content: "a wiki page", Meta: map[string]any{"kind": "page", "slug": "entity/alpha"}},+ }+ got := navInputFromProducts("kb1", products)+ if len(got) != 0 {+ t.Fatalf("wiki products must not feed nav, got %d inputs", len(got))+ }+}++// TestNavInputFromProducts_EmptySummaryDropped covers the guard: a doc whose+// tree/structure summary is empty (no root / no graph) is skipped, not fed empty.+func TestNavInputFromProducts_EmptySummaryDropped(t *testing.T) {+ products := []kccommon.Product{+ // tree root with empty content -> skip.+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree, Meta: map[string]any{"kind": "root"}},+ // structure graph with no entity descriptions -> skip.+ {DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,+ Content: `{"entities":[]}`, Meta: map[string]any{"kind": "graph"}},+ }+ got := navInputFromProducts("kb1", products)+ if len(got) != 0 {+ t.Fatalf("empty-summary products must be dropped, got %d inputs", len(got))+ }+}++// TestProductsForVariants_GatesDispatch covers the A0-4 per-variant gate: a doc+// whose event carries only ["wiki"] must NOT have its stale tree/structure+// doc-level products leak into the nav/structure paths. Empty variants (legacy+// events) keep everything.+func TestProductsForVariants_GatesDispatch(t *testing.T) {+ products := []kccommon.Product{+ {DocID: "d1", Variant: kccommon.VariantTree, Meta: map[string]any{"kind": "root"}, Content: "tree summary"},+ {DocID: "d1", Variant: kccommon.VariantStructure, Meta: map[string]any{"kind": "graph"}, Content: `{"entities":[]}`},+ {DocID: "d1", Variant: kccommon.VariantWiki, Meta: map[string]any{"kind": "page"}, Content: "wiki page"},+ }+ // A wiki-only event must drop the tree/structure products.+ got := productsForVariants(products, []string{string(kccommon.VariantWiki)})+ if len(got) != 1 || got[0].Variant != kccommon.VariantWiki {+ t.Fatalf("wiki-only event: want exactly the wiki product, got %d", len(got))+ }+ // A tree-only event drops structure + wiki.+ got = productsForVariants(products, []string{string(kccommon.VariantTree)})+ if len(got) != 1 || got[0].Variant != kccommon.VariantTree {+ t.Fatalf("tree-only event: want exactly the tree product, got %d", len(got))+ }+ // An empty variant set (legacy event / no variants) keeps everything.+ if got = productsForVariants(products, nil); len(got) != 3 {+ t.Fatalf("empty variants: want all 3 products kept, got %d", len(got))+ }+}++// TestPageIndexSummary_ConcatenatesAndSkipsEmpty verifies pageIndexSummary folds+// entity descriptions and skips empty ones.+func TestPageIndexSummary_ConcatenatesAndSkipsEmpty(t *testing.T) {+ got := pageIndexSummary(`{"entities":[{"name":"A","description":"one thing"},{"name":"B","description":""}]}`)+ if !strings.Contains(got, "A: one thing") {+ t.Errorf("summary = %q, want entity A", got)+ }+ if strings.Contains(got, "B") {+ t.Errorf("summary should skip empty-description entity, got %q", got)+ }+}diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 809605b86f..6861ba35db 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -79,6 +79,12 @@ var compiledSelectFields = []string{"name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd","slug_kwd", "type","kc_kind", "create_timestamp_flt", "create_time",+ "compilation_template_kind_kwd",+ // The product kind discriminator for structure/tree: the component stores it+ // under knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd+ // (tree: root/summary). Without these the reader cannot restore Meta["kind"]+ // and the dataset-nav dispatch would skip structure/tree products (B2).+ "knowledge_graph_kwd", "raptor_kwd",}// wikiSelectFields are the additional columns a wiki page carries (beyond@@ -136,8 +142,8 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID strcontinue}// Reverse-map the row's compile_kwd; reject dirty/unknown kinds- // (kwdToVariant error) so a malformed row never loads as a product.- rowVariant, verr := kwdToVariant(asString(c["compile_kwd"]))+ // (KwdToVariant error) so a malformed row never loads as a product.+ rowVariant, verr := KwdToVariant(asString(c["compile_kwd"]))if verr != nil {continue}@@ -176,7 +182,7 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID str// content_with_weight) and the embedding from the q_<dim>_vec column.//// expect is the variant the caller is querying for. The stored compile_kwd is-// reverse-mapped via kwdToVariant and compared against expect; a mismatch (or+// reverse-mapped via KwdToVariant and compared against expect; a mismatch (or// an unknown/dirty compile_kwd that does not map to any known variant) causes// the row to be skipped. This is the canonical dirty-row contract promised by// the plan: we never rely on the raw string equality alone, so unknown kinds@@ -196,9 +202,9 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo// reverse-mapped consistently instead of being rejected as a dirty row.variant := asString(c["compile_kwd"])// Dirty-row contract: the raw compile_kwd must reverse-map to the expected- // variant. An unknown/dirty kwd (kwdToVariant error) or a mapped variant+ // variant. An unknown/dirty kwd (KwdToVariant error) or a mapped variant// that differs from expect is rejected.- mapped, err := kwdToVariant(variant)+ mapped, err := KwdToVariant(variant)if err != nil || mapped != expect {return kccommon.Product{}, false}@@ -253,7 +259,17 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommoif v, ok := c["type"].(string); ok && v != "" {meta["type"] = v}- if _, ok := meta["kind"]; !ok {+ // Restore the structure/tree product kind. The component stores it under+ // knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd (tree:+ // root/summary), and the dataset-nav dispatch (B2) keys off Meta["kind"] to+ // pick the root/graph summary. Prefer these over the generic entity default.+ // Use asString (not a bare type assertion): the engine may return a+ // list-wrapped keyword column (review Major).+ if v := asString(c["knowledge_graph_kwd"]); v != "" {+ meta["kind"] = v+ } else if v := asString(c["raptor_kwd"]); v != "" {+ meta["kind"] = v+ } else if _, ok := meta["kind"]; !ok {if _, hasName := meta["name"]; hasName {meta["kind"] = "entity"}@@ -265,7 +281,7 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo// wiki_section -> "section". This fix is what stops the processBatch// "Meta.kind==page" filter from deleting every wiki page (previously kind// was empty for wiki pages that had no entity/relation endpoint).- if v, ok := c["kc_kind"].(string); ok && v != "" {+ if v := asString(c["kc_kind"]); v != "" {meta["kind"] = v} else if variant == compileKwdWikiPage {meta["kind"] = "page"@@ -290,11 +306,16 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo}vec, _ := kccommon.VectorFromChunkMap(c, 0)+ // Restore the authoritative template kind (compilation_template_kind_kwd)+ // so callers (e.g. RebuildDataset's variant recovery, B1a) can map it back+ // via KindToVariant instead of re-deriving from the ambiguous compile_kwd.+ kind := asString(c["compilation_template_kind_kwd"])return kccommon.Product{ID: id,DocID: docID,TenantID: tenant,Variant: expect,+ Kind: kind,Content: content,Vector: vec,Meta: meta,@@ -353,7 +374,7 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, vari// The KNN Filter already scopes compile_kwd, but a foreign/legacy row that// slipped past it must not poison the merge candidate. Reject by the// reverse-mapped variant (the dirty-row contract): productFromChunkMap- // validates kwdToVariant(c) == variant and drops dirty/unknown kinds. The+ // validates KwdToVariant(c) == variant and drops dirty/unknown kinds. The// raw-keyword check below is a fast pre-filter before the full product// reconstruction.if asString(c["compile_kwd"]) != expectKwd {diff --git a/internal/ingestion/knowledge_compile/rebuild_variants_test.go b/internal/ingestion/knowledge_compile/rebuild_variants_test.gonew file mode 100644index 0000000000..cef9c7b949--- /dev/null+++ b/internal/ingestion/knowledge_compile/rebuild_variants_test.go@@ -0,0 +1,71 @@+package knowledge_compile++import (+ "context"+ "reflect"+ "testing"++ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// fakeReader is a Reader double that returns a canned product set for+// recoverDocVariants tests.+type fakeReader struct {+ products []kccommon.Product+}++func (f *fakeReader) LoadDocProducts(context.Context, string, string, string) ([]kccommon.Product, error) {+ return f.products, nil+}++func (f *fakeReader) SearchSimilar(context.Context, string, string, kccommon.Variant, []float64, int, float64) (kccommon.Product, float64, error) {+ return kccommon.Product{}, 0, nil+}++// TestRecoverDocVariants_AuthoritativeKind covers B1a/O2a: the authoritative+// Product.Kind (compilation_template_kind_kwd) is mapped through KindToVariant;+// results are sorted/deduped.+func TestRecoverDocVariants_AuthoritativeKind(t *testing.T) {+ c := &Consumer{reader: &fakeReader{products: []kccommon.Product{+ {DocID: "d1", Kind: "structure"},+ {DocID: "d1", Kind: "tree"},+ // duplicate variant, deduped+ {DocID: "d1", Kind: "structure"},+ }}}+ got, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1")+ if err != nil {+ t.Fatalf("recoverDocVariants error: %v", err)+ }+ want := []string{string(kccommon.VariantStructure), string(kccommon.VariantTree)}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("recoverDocVariants = %v, want %v", got, want)+ }+}++// TestRecoverDocVariants_UnknownKindHardFails covers O2a: a whitelist-out+// authoritative kind aborts recovery (returns an error) so the rebuild does not+// proceed with an incomplete variant set.+func TestRecoverDocVariants_UnknownKindHardFails(t *testing.T) {+ c := &Consumer{reader: &fakeReader{products: []kccommon.Product{+ {DocID: "d1", Kind: "structure"},+ {DocID: "d1", Kind: "garbage"},+ }}}+ if _, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1"); err == nil {+ t.Fatal("recoverDocVariants must hard-fail on an unknown authoritative kind (O2a)")+ }+}++// TestRecoverDocVariants_FallbackVariant covers B1a: a product without an+// authoritative kind falls back to its reverse-mapped variant.+func TestRecoverDocVariants_FallbackVariant(t *testing.T) {+ c := &Consumer{reader: &fakeReader{products: []kccommon.Product{+ {DocID: "d1", Variant: kccommon.VariantWiki},+ }}}+ got, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1")+ if err != nil {+ t.Fatalf("recoverDocVariants error: %v", err)+ }+ if len(got) != 1 || got[0] != string(kccommon.VariantWiki) {+ t.Fatalf("recoverDocVariants fallback = %v, want [wiki]", got)+ }+}diff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.goindex bf9d7f136b..c2c4f14ad1 100644--- a/internal/ingestion/knowledge_compile/scheduler.go+++ b/internal/ingestion/knowledge_compile/scheduler.go@@ -57,9 +57,17 @@ const (// can re-apply the same tombstone handling as the broker-based design without// re-reading the queue. The backlog is an ordered log; the last event for a// given doc (by append order) wins, so no sequence number is needed.+//+// Variants records the compile types (tree/structure/wiki/mindmap) produced by+// the doc-level compile for this doc, so the consumer can group a claimed batch+// into per-variant sub-batches and dispatch each to its own dataset-level+// compile path. It is empty (nil) for deleted events and for events published+// before this field existed; the consumer treats a nil/empty Variants as+// "unknown/legacy" and falls back to the legacy unified path.type BacklogEntry struct {- DocID string `json:"doc_id"`- EventType string `json:"event_type"`+ DocID string `json:"doc_id"`+ EventType string `json:"event_type"`+ Variants []string `json:"variants,omitempty"`}// ClaimResult is returned by Scheduler.Claim: the KB's tenant plus the closed@@ -83,7 +91,9 @@ type Publisher interface {// Publish records one doc event in the dataset's durable backlog and wakes// idle workers. It is transactional so concurrent publishers do not clobber// each other's backlog, and it always pairs the append with a notify.- Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error+ // variants carries the doc-level compile types (tree/structure/wiki/mindmap)+ // for completed events; it is nil for deleted events.+ Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error}// Claimer is the consumer-side role (Option E §11.5). A cluster of competing@@ -203,8 +213,8 @@ func (s *mysqlScheduler) Provision(ctx context.Context) error {// workers. The append is transactional (concurrent publishers do not clobber// each other's backlog), and the notify is always paired with the append so a// producer never needs a separate Notify call.-func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error {- return s.publish(ctx, tenantID, datasetID, docID, eventType)+func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {+ return s.publish(ctx, tenantID, datasetID, docID, eventType, variants)}// loadRow fetches the dataset scheduling row (no lock). Returns nil (no error)@@ -220,11 +230,11 @@ func (s *mysqlScheduler) loadRow(ctx context.Context, datasetID string) (*entityreturn &row, nil}-func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error {+func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {if s.db == nil {return nil}- entry := BacklogEntry{DocID: docID, EventType: eventType}+ entry := BacklogEntry{DocID: docID, EventType: eventType, Variants: variants}// KnowledgeCompileDataset.dataset_id is the PRIMARY KEY, so the SELECT ... FOR// UPDATE below also takes an InnoDB gap lock for a not-yet-existing dataset;// concurrent publishers for the same dataset serialize on that lock and the@@ -642,17 +652,23 @@ func (s *mysqlScheduler) SubscribeNotify(ctx context.Context) (<-chan string, er// doc_id + event_type). The batch is the exact set the worker claimed, so// this is a precise removal, not a "clear all" — any inflight added by a// concurrent claim is preserved.+//+// The key is doc_id+"\x00"+event_type: BacklogEntry now carries a Variants+// slice which is not comparable, so it cannot be used as a Go map key. Variants+// deliberately does not participate in the removal match — Ack still removes by+// doc_id + event_type so an entry whose Variants differ from the inflight copy+// is still removed exactly once.func removeEntries(inflight, batch []BacklogEntry) []BacklogEntry {if len(batch) == 0 {return inflight}- drop := make(map[BacklogEntry]bool, len(batch))+ drop := make(map[string]bool, len(batch))for _, e := range batch {- drop[e] = true+ drop[e.DocID+"\x00"+e.EventType] = true}out := make([]BacklogEntry, 0, len(inflight))for _, e := range inflight {- if drop[e] {+ if drop[e.DocID+"\x00"+e.EventType] {continue}out = append(out, e)@@ -699,11 +715,11 @@ func NewFakeScheduler() *FakeScheduler {func (f *FakeScheduler) Provision(_ context.Context) error { return nil }// Publish appends one doc event and pushes a notify (same contract as MySQL).-func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string) error {- return f.publish(tenantID, datasetID, docID, eventType)+func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {+ return f.publish(tenantID, datasetID, docID, eventType, variants)}-func (f *FakeScheduler) publish(tenantID, datasetID, docID, eventType string) error {+func (f *FakeScheduler) publish(tenantID, datasetID, docID, eventType string, variants []string) error {f.mu.Lock()defer f.mu.Unlock()r, ok := f.rows[datasetID]@@ -714,7 +730,7 @@ func (f *FakeScheduler) publish(tenantID, datasetID, docID, eventType string) erif r.tenant == "" {r.tenant = tenantID}- r.backlog = append(r.backlog, BacklogEntry{DocID: docID, EventType: eventType})+ r.backlog = append(r.backlog, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})// Mirror the MySQL scheduler: surface pending unless a worker is running.if r.state != DatasetStateRunning {r.state = DatasetStatePendingdiff --git a/internal/ingestion/knowledge_compile/scheduler_test.go b/internal/ingestion/knowledge_compile/scheduler_test.gonew file mode 100644index 0000000000..dc413dee21--- /dev/null+++ b/internal/ingestion/knowledge_compile/scheduler_test.go@@ -0,0 +1,106 @@+package knowledge_compile++import (+ "encoding/json"+ "reflect"+ "testing"+)++// TestRemoveEntriesMatchesByDocAndEventType locks the A0-1 contract: BacklogEntry+// now carries a Variants slice (not comparable), so removeEntries must key on+// DocID+EventType only and must not be affected by Variants. Two entries with the+// same doc_id+event_type but different variants must collapse to a single removal,+// and an entry whose Variants differ from the inflight copy must still be removed+// exactly once.+func TestRemoveEntriesMatchesByDocAndEventType(t *testing.T) {+ inflight := []BacklogEntry{+ {DocID: "d1", EventType: "doc_completed", Variants: []string{"tree", "wiki"}},+ {DocID: "d2", EventType: "doc_completed", Variants: []string{"structure"}},+ {DocID: "d3", EventType: "doc_deleted"},+ }++ // Batch claims d1 (with a variants subset different from inflight) and d3.+ batch := []BacklogEntry{+ {DocID: "d1", EventType: "doc_completed", Variants: []string{"wiki"}},+ {DocID: "d3", EventType: "doc_deleted"},+ }++ out := removeEntries(inflight, batch)++ if len(out) != 1 || out[0].DocID != "d2" {+ t.Fatalf("removeEntries should keep only d2, got %+v", out)+ }+}++// TestRemoveEntriesEmptyBatch verifies removeEntries with an empty batch is a no-op.+func TestRemoveEntriesEmptyBatch(t *testing.T) {+ inflight := []BacklogEntry{{DocID: "d1", EventType: "doc_completed", Variants: []string{"tree"}}}+ out := removeEntries(inflight, nil)+ if !reflect.DeepEqual(out, inflight) {+ t.Fatalf("empty batch should preserve inflight, got %+v", out)+ }+}++// TestBacklogEntryJSONBackwardCompat locks the A0-1 JSON contract: an old backlog+// row serialized without the "variants" field must unmarshal with Variants == nil+// (empty, not an error), which the consumer treats as "legacy/unknown" and falls+// back to the unified path.+func TestBacklogEntryJSONBackwardCompat(t *testing.T) {+ old := []byte(`{"doc_id":"d1","event_type":"doc_completed"}`)+ var e BacklogEntry+ if err := json.Unmarshal(old, &e); err != nil {+ t.Fatalf("unmarshal legacy backlog: %v", err)+ }+ if e.DocID != "d1" || e.EventType != "doc_completed" {+ t.Fatalf("legacy fields not restored: %+v", e)+ }+ if e.Variants != nil {+ t.Fatalf("legacy backlog should unmarshal Variants as nil, got %v", e.Variants)+ }++ // A completed event with variants round-trips intact.+ withVariants := []byte(`{"doc_id":"d1","event_type":"doc_completed","variants":["tree","wiki"]}`)+ if err := json.Unmarshal(withVariants, &e); err != nil {+ t.Fatalf("unmarshal variants backlog: %v", err)+ }+ if !reflect.DeepEqual(e.Variants, []string{"tree", "wiki"}) {+ t.Fatalf("variants not restored: %+v", e.Variants)+ }+ // omitempty: a nil Variants serializes without the key (matches legacy format).+ enc, err := json.Marshal(BacklogEntry{DocID: "d2", EventType: "doc_deleted"})+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ var decoded map[string]any+ if err := json.Unmarshal(enc, &decoded); err != nil {+ t.Fatalf("re-marshal decode: %v", err)+ }+ if _, hasVariants := decoded["variants"]; hasVariants {+ t.Fatalf("nil Variants should be omitted, got %s", enc)+ }+}++// TestFakeSchedulerPublishCarriesVariants locks the A0-2 contract: FakeScheduler+// records the variants on the backlog entry, so the consumer sees them.+func TestFakeSchedulerPublishCarriesVariants(t *testing.T) {+ f := NewFakeScheduler()+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"tree", "structure"}); err != nil {+ t.Fatalf("publish: %v", err)+ }+ if err := f.Publish(t.Context(), "t1", "kb1", "d2", string(EventTypeDeleted), nil); err != nil {+ t.Fatalf("publish deleted: %v", err)+ }+ res, ok, err := f.Claim(t.Context(), "kb1")+ if err != nil || !ok {+ t.Fatalf("claim: ok=%v err=%v", ok, err)+ }+ if len(res.Entries) != 2 {+ t.Fatalf("want 2 backlog entries, got %d", len(res.Entries))+ }+ if !reflect.DeepEqual(res.Entries[0].Variants, []string{"tree", "structure"}) {+ t.Fatalf("completed variants not carried: %+v", res.Entries[0])+ }+ if res.Entries[1].Variants != nil {+ t.Fatalf("deleted event should carry nil variants, got %+v", res.Entries[1])+ }+}diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.gonew file mode 100644index 0000000000..764a9b8b9c--- /dev/null+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go@@ -0,0 +1,204 @@+package knowledge_compile++import (+ "context"+ "strings"+ "testing"++ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// fakeWriter is a Writer double that records the last WriteMergedStructure+// buckets so a test can assert the grouping behavior without an engine.+type fakeWriter struct {+ buckets []StructureBucket+}++func (f *fakeWriter) WriteMerged(context.Context, string, string, []kccommon.Product) error {+ return nil+}+func (f *fakeWriter) WriteMergedStructure(_ context.Context, _, _ string, buckets []StructureBucket) error {+ f.buckets = buckets+ return nil+}+func (f *fakeWriter) DeleteMergedForVariant(context.Context, string, string, []kccommon.Variant) error {+ return nil+}+func (f *fakeWriter) DeleteStructureForDocs(context.Context, string, string, []string) error {+ return nil+}+func (f *fakeWriter) DeleteMerged(context.Context, string, string) error { return nil }+func (f *fakeWriter) DeleteDocLevelForDocs(context.Context, string, string, []string) error {+ return nil+}+func (f *fakeWriter) StripMergedSources(context.Context, string, string, []string) error {+ return nil+}+func (f *fakeWriter) ProjectWikiGraph(context.Context, string, string) error { return nil }+func (f *fakeWriter) DropWikiGraph(context.Context, string, string) error { return nil }++// TestMergeStructureDataset_GroupsByNameType covers G1: structure products are+// bucketed by (name, type), descriptions folded, source ids unioned, and only+// scope_kwd="dataset" structure rows written.+func TestMergeStructureDataset_GroupsByNameType(t *testing.T) {+ c := &Consumer{writer: &fakeWriter{}}+ products := []kccommon.Product{+ {Variant: kccommon.VariantStructure, DocID: "d1",+ Content: "Engine desc A",+ Meta: map[string]any{"name": "Engine", "entity_type": "component", "source_chunk_ids": []string{"c1"}}},+ {Variant: kccommon.VariantStructure, DocID: "d2",+ Content: "Engine desc B",+ Meta: map[string]any{"name": "Engine", "entity_type": "component", "source_chunk_ids": []string{"c2"}}},+ // different type -> separate bucket+ {Variant: kccommon.VariantStructure, DocID: "d3",+ Content: "Fuel desc",+ Meta: map[string]any{"name": "Fuel", "entity_type": "substance", "source_chunk_ids": []string{"c3"}}},+ // non-structure product ignored+ {Variant: kccommon.VariantWiki, DocID: "d4", Content: "wiki page",+ Meta: map[string]any{"name": "Ignored", "kind": "page"}},+ }+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {+ t.Fatalf("mergeStructureDataset: %v", err)+ }+ fw := c.writer.(*fakeWriter)+ if len(fw.buckets) != 2 {+ t.Fatalf("want 2 buckets (Engine + Fuel), got %d", len(fw.buckets))+ }+ var engineBucket, fuelBucket *StructureBucket+ for i := range fw.buckets {+ switch fw.buckets[i].Name {+ case "Engine":+ engineBucket = &fw.buckets[i]+ case "Fuel":+ fuelBucket = &fw.buckets[i]+ }+ }+ if engineBucket == nil || fuelBucket == nil {+ t.Fatalf("missing expected buckets: %+v", fw.buckets)+ }+ if engineBucket.Type != "component" {+ t.Errorf("Engine type = %q, want component", engineBucket.Type)+ }+ if !strings.Contains(engineBucket.Description, "Engine desc A") || !strings.Contains(engineBucket.Description, "Engine desc B") {+ t.Errorf("Engine descriptions not folded: %q", engineBucket.Description)+ }+ if len(engineBucket.SourceDocIDs) != 2 {+ t.Errorf("Engine source doc union = %v, want [d1 d2]", engineBucket.SourceDocIDs)+ }+}++// TestMergeStructureDataset_RelationsNotDropped covers review issue 3: structure+// relation products (kind=relation with from/to, no name) must be folded into a+// dataset-level relation bucket instead of being silently dropped.+func TestMergeStructureDataset_RelationsNotDropped(t *testing.T) {+ c := &Consumer{writer: &fakeWriter{}}+ products := []kccommon.Product{+ {Variant: kccommon.VariantStructure, DocID: "d1", Content: "rel desc",+ Meta: map[string]any{"kind": "relation", "from": "A", "to": "B", "source_chunk_ids": []string{"c1"}}},+ {Variant: kccommon.VariantStructure, DocID: "d2", Content: "rel desc 2",+ Meta: map[string]any{"kind": "relation", "from": "A", "to": "B", "source_chunk_ids": []string{"c2"}}},+ }+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {+ t.Fatalf("mergeStructureDataset: %v", err)+ }+ fw := c.writer.(*fakeWriter)+ if len(fw.buckets) != 1 {+ t.Fatalf("want 1 relation bucket, got %d", len(fw.buckets))+ }+ b := fw.buckets[0]+ if b.Type != "relation" || b.FromEntity != "A" || b.ToEntity != "B" {+ t.Errorf("relation bucket = %+v, want Type=relation From=A To=B", b)+ }+ if len(b.SourceDocIDs) != 2 {+ t.Errorf("relation source doc union = %v, want [d1 d2]", b.SourceDocIDs)+ }+}++// TestDatasetLevelStructureID_StableAndCaseInsensitive covers G1: the id is+// deterministic and case-insensitive on name so merges hit the same row.+func TestDatasetLevelStructureID_StableAndCaseInsensitive(t *testing.T) {+ a := datasetLevelStructureID("t1", "kb1", "Engine", "component")+ b := datasetLevelStructureID("t1", "kb1", "engine", "component")+ if a != b {+ t.Errorf("structure id should be case-insensitive on name: %q vs %q", a, b)+ }+ if a == datasetLevelStructureID("t1", "kb1", "Fuel", "component") {+ t.Errorf("different names must yield different ids")+ }+}++// TestKwdToVariant_MapsStructureSubKinds covers B1a: structure doc-level products+// stamp their inferred compile kind verbatim (hypergraph/list/set/timeline/+// page_index/...), and KwdToVariant must reverse-map each to VariantStructure so+// the reader does NOT drop them before the dataset-level merge / nav dispatch.+func TestKwdToVariant_MapsStructureSubKinds(t *testing.T) {+ for _, kwd := range []string{+ "hypergraph", "list", "set", "timeline", "page_index",+ "session_essence", "session_graph", "knowledge_graph", "graph", "structure",+ } {+ v, err := KwdToVariant(kwd)+ if err != nil {+ t.Errorf("KwdToVariant(%q) unexpected error: %v", kwd, err)+ continue+ }+ if v != kccommon.VariantStructure {+ t.Errorf("KwdToVariant(%q) = %q, want structure", kwd, v)+ }+ }+}++// TestKwdToVariant_RejectsUnknown still hard-fails on a non-whitelisted kwd+// (O2a): a malformed or unknown compile_kwd is rejected, not silently folded.+func TestKwdToVariant_RejectsUnknown(t *testing.T) {+ for _, kwd := range []string{"garbage", "artifact_page", ""} {+ if _, err := KwdToVariant(kwd); err == nil {+ t.Errorf("KwdToVariant(%q) should error", kwd)+ }+ }+}++// TestProductFromChunkMap_RestoresStructureTreeKind covers the second-round fix:+// the reader must restore Meta["kind"] for structure (from knowledge_graph_kwd)+// and tree (from raptor_kwd) products so the dataset-nav dispatch (B2) can pick+// the graph/root summary instead of skipping them.+func TestProductFromChunkMap_RestoresStructureTreeKind(t *testing.T) {+ // structure graph product: compile_kwd is the inferred sub-kind "list",+ // kind is stored in knowledge_graph_kwd.+ structRow := map[string]interface{}{+ "id": "s1",+ "doc_id": "d1",+ "compile_kwd": "list",+ "kc_payload": `{"entities":[]}`,+ "knowledge_graph_kwd": "graph",+ "name_kwd": "engine",+ "source_chunk_ids": []interface{}{"c1"},+ "source_doc_ids": []interface{}{"d1"},+ "compilation_template_kind_kwd": "list",+ }+ sp, ok := productFromChunkMap(structRow, "t1", kccommon.VariantStructure)+ if !ok {+ t.Fatal("structure row should reconstruct")+ }+ if kind, _ := sp.Meta["kind"].(string); kind != "graph" {+ t.Errorf("structure Meta.kind = %q, want graph (so nav dispatch can pick it)", kind)+ }++ // tree root product: kind stored in raptor_kwd.+ treeRow := map[string]interface{}{+ "id": "t1",+ "doc_id": "d1",+ "compile_kwd": "tree",+ "kc_payload": "overall summary",+ "raptor_kwd": "root",+ "source_chunk_ids": []interface{}{"c1"},+ "source_doc_ids": []interface{}{"d1"},+ "compilation_template_kind_kwd": "tree",+ }+ tp, ok := productFromChunkMap(treeRow, "t1", kccommon.VariantTree)+ if !ok {+ t.Fatal("tree row should reconstruct")+ }+ if kind, _ := tp.Meta["kind"].(string); kind != "root" {+ t.Errorf("tree Meta.kind = %q, want root (so nav dispatch can pick it)", kind)+ }+}diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 97d2450540..88cc9514ef 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -60,6 +60,23 @@ type Writer interface {ProjectWikiGraph(ctx context.Context, tenant, kb string) error// DropWikiGraph deletes every wiki_entity / wiki_relation row for the dataset.DropWikiGraph(ctx context.Context, tenant, kb string) error+ // WriteMergedStructure writes the dataset-level structure merged rows+ // (scope_kwd="dataset") for a KB, one row per (name, type) bucket, carrying+ // the folded descriptions and the union of source docs/chunks (G1/G4).+ WriteMergedStructure(ctx context.Context, tenant, kb string, buckets []StructureBucket) error+ // DeleteStructureForDocs removes dataset-level structure rows (scope_kwd=+ // "dataset", compile_kwd="structure") that reference a deleted doc and no+ // longer have any remaining source doc (G3 ghost cleanup). Rows that still+ // have other source docs are kept; their source_doc_ids are NOT stripped here+ // (StripMergedSources handles the union-preserving edit).+ DeleteStructureForDocs(ctx context.Context, tenant, kb string, deletedDocIDs []string) error+ // DeleteMergedForVariant removes the dataset-level merged rows for a KB whose+ // compile type is in variants (B4): structure (scope_kwd="dataset"),+ // wiki (compile_kwd wiki_page/wiki_section), and nav (compile_kwd+ // dataset_nav) as applicable. A full rebuild clears every variant the+ // consumer manages (B1b: fixed full-set, empty set also clears all) so+ // removed-template ghosts cannot survive.+ DeleteMergedForVariant(ctx context.Context, tenant, kb string, variants []kccommon.Variant) error// DeleteMerged removes the dataset-level merged rows for a KB so an// incremental build can start from a clean slate. The structural filter// deletes only rows produced by the dataset-level merge — kb_id == kb AND@@ -70,6 +87,22 @@ type Writer interface {DeleteMerged(ctx context.Context, tenant, kb string) error}+// StructureBucket is one dataset-level structure merge unit (G1/G4): a group of+// a structure entity (Name/Type) OR a relation (FromEntity/ToEntity) with its+// folded descriptions and the union of source docs/chunks. It is written as a+// scope_kwd="dataset" row. Relations carry FromEntity/ToEntity instead of Name.+type StructureBucket struct {+ Name string+ Type string+ Description string // folded entity descriptions+ SourceDocIDs []string // union of source doc ids+ SourceChunkIDs []string // union of source chunk ids+ Vector []float32+ VecCount int // number of vectors folded into Vector (for true mean)+ FromEntity string // relation only+ ToEntity string // relation only+}+// engineWriter persists dataset-level merged products through the global// DocEngine (§11.7). Like engineReader, it depends on the process-wide DocEngine// obtained via engine.Get(); the storage schema lives behind the engine@@ -167,6 +200,218 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, producreturn runCompilerJobs(ctx, jobs)}+// WriteMergedStructure writes the dataset-level structure merged rows for a KB+// (G1/G4). Each StructureBucket is a scope_kwd="dataset" row with a stable+// dataset-level id keyed on (name, type), the folded description, the union of+// source docs/chunks, and the bucket vector. Rows are available_int=1 so the+// dataset-level structure index is searchable, and compile_kwd="structure".+func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb string, buckets []StructureBucket) error {+ if len(buckets) == 0 {+ return nil+ }+ eng := w.eng+ if eng == nil {+ eng = engine.Get()+ }+ if eng == nil {+ return nil+ }+ baseName := fmt.Sprintf("ragflow_%s", tenant)+ now := time.Now()+ // Read-modify-write: an incremental batch must not drop the source docs/chunks+ // an earlier batch already accumulated for a (name,type) bucket, so load the+ // existing dataset rows by their stable id and union their sources (review+ // issue 3 / #3 Major).+ existing := map[string]StructureBucket{}+ {+ ids := make([]string, 0, len(buckets))+ for _, b := range buckets {+ if b.Name == "" {+ continue+ }+ ids = append(ids, datasetLevelStructureID(tenant, kb, b.Name, b.Type))+ }+ if len(ids) > 0 {+ res, err := eng.Search(ctx, &types.SearchRequest{+ IndexNames: []string{baseName},+ KbIDs: []string{kb},+ SelectFields: []string{"id", "source_doc_ids", "source_chunk_ids"},+ Filter: map[string]interface{}{"kb_id": kb, "id": ids},+ Limit: len(ids),+ })+ if err != nil {+ return fmt.Errorf("structure merge read-modify-write load: %w", err)+ }+ for _, c := range res.Chunks {+ id, _ := c["id"].(string)+ if id == "" {+ continue+ }+ existing[id] = StructureBucket{+ SourceDocIDs: firstStringSlice(c["source_doc_ids"]),+ SourceChunkIDs: firstStringSlice(c["source_chunk_ids"]),+ }+ }+ }+ }+ rows := make([]map[string]interface{}, 0, len(buckets))+ for _, b := range buckets {+ desc := strings.TrimSpace(b.Description)+ if desc == "" {+ continue+ }+ if b.Name == "" {+ continue+ }+ bid := datasetLevelStructureID(tenant, kb, b.Name, b.Type)+ // Union the current batch's sources with any already-accumulated ones.+ if prev, ok := existing[bid]; ok {+ b.SourceDocIDs = appendUnique(b.SourceDocIDs, prev.SourceDocIDs)+ b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, prev.SourceChunkIDs)+ }+ row := map[string]interface{}{+ // Stable dataset-level id keyed on the (name, type) or (from, to) bucket.+ "id": bid,+ "doc_id": kb,+ "tenant_id": tenant,+ "kb_id": kb,+ "available_int": 1,+ "compile_kwd": compileKwdStructure,+ "scope_kwd": "dataset",+ "content_with_weight": desc,+ "kc_payload": desc,+ "source_doc_ids": b.SourceDocIDs,+ "source_chunk_ids": b.SourceChunkIDs,+ "create_time": now.Format("2006-01-02 15:04:05"),+ "create_timestamp_flt": float64(now.Unix()),+ }+ if b.FromEntity != "" || b.ToEntity != "" {+ // relation row: carries from/to entities; kind=relation, no name_kwd.+ row["type_kwd"] = "relation"+ row["from_entity_kwd"] = b.FromEntity+ row["to_entity_kwd"] = b.ToEntity+ } else {+ row["type_kwd"] = "entity"+ row["name_kwd"] = b.Name+ row["entity_type_kwd"] = b.Type+ }+ if len(b.Vector) > 0 {+ row["q_"+fmt.Sprintf("%d", len(b.Vector))+"_vec"] = f32ToF64Slice(b.Vector)+ }+ rows = append(rows, row)+ }+ if len(rows) == 0 {+ return nil+ }+ // Insert in bounded batches, matching writeMergedBatchSize.+ jobs := make([]CompilerJob, 0, (len(rows)+writeMergedBatchSize-1)/writeMergedBatchSize)+ for start := 0; start < len(rows); start += writeMergedBatchSize {+ end := start + writeMergedBatchSize+ if end > len(rows) {+ end = len(rows)+ }+ batch := rows[start:end]+ jobs = append(jobs, func() error {+ _, err := eng.InsertChunks(ctx, batch, baseName, kb)+ return err+ })+ }+ return runCompilerJobs(ctx, jobs)+}++// DeleteStructureForDocs removes dataset-level structure rows whose source docs+// are all gone (G3, mirroring Python dataset_structure_merger._cleanup_deleted_docs).+// A structure dataset row that still has any non-deleted source doc survives; a+// row whose source_doc_ids are a subset of the deleted set is a ghost and is+// removed. This is best-effort in the sense that rows without any source_doc_ids+// are untouched (they may predate source tracking).+func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb string, deletedDocIDs []string) error {+ if len(deletedDocIDs) == 0 {+ return nil+ }+ eng := w.eng+ if eng == nil {+ eng = engine.Get()+ }+ if eng == nil {+ return nil+ }+ baseName := fmt.Sprintf("ragflow_%s", tenant)+ deleted := make(map[string]bool, len(deletedDocIDs))+ for _, d := range deletedDocIDs {+ deleted[d] = true+ }+ // Page through structure dataset rows (mirroring StripMergedSources) so a+ // dataset with more than one page of rows does not leave ghosts past the cap+ // surviving silently (review Minor).+ const pageSize = 500+ var ghostIDs []string+ for offset := 0; ; offset += pageSize {+ res, err := eng.Search(ctx, &types.SearchRequest{+ IndexNames: []string{baseName},+ KbIDs: []string{kb},+ SelectFields: []string{"id", "source_doc_ids"},+ Filter: map[string]interface{}{"kb_id": kb, "scope_kwd": "dataset", "compile_kwd": compileKwdStructure},+ Offset: offset,+ Limit: pageSize,+ })+ if err != nil {+ return fmt.Errorf("structure ghost scan: %w", err)+ }+ if len(res.Chunks) == 0 {+ break+ }+ for _, c := range res.Chunks {+ id, _ := c["id"].(string)+ if id == "" {+ continue+ }+ srcs := firstStringSlice(c["source_doc_ids"])+ if len(srcs) == 0 {+ continue // no source tracking; not safe to declare a ghost+ }+ allGone := true+ for _, s := range srcs {+ if !deleted[s] {+ allGone = false+ break+ }+ }+ if allGone {+ ghostIDs = append(ghostIDs, id)+ }+ }+ if len(res.Chunks) < pageSize {+ break+ }+ }+ if len(ghostIDs) == 0 {+ return nil+ }+ _, err := eng.DeleteChunks(ctx, map[string]interface{}{"id": ghostIDs, "kb_id": kb}, baseName, kb)+ if err != nil {+ return fmt.Errorf("structure ghost cleanup: %w", err)+ }+ return nil+}++// datasetLevelStructureID builds the stable dataset-level id for a structure+// bucket, keyed on (name, type). It must be deterministic so an incremental+// merge read-modify-writes the same row (and a rebuild clean removes it).+func datasetLevelStructureID(tenant, kb, name, typ string) string {+ return "dataset_structure_" + hashStr(tenant+"\x00"+kb+"\x00"+strings.ToLower(name)+"\x00"+typ)+}++// f32ToF64Slice converts a float32 vector to float64 for the engine's dense+// vector column (the engine stores q_*_vec as float64).+func f32ToF64Slice(v []float32) []float64 {+ out := make([]float64, len(v))+ for i, x := range v {+ out[i] = float64(x)+ }+ return out+}+// mergedChunkMap builds the chunk-index document for a dataset-level merged// product. It uses the dataset-level idempotency key (§11.6) as `id`, never the// per-doc key, and is always available_int=1 (searchable).@@ -181,11 +426,16 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommsrcDocIDs := metaStringSlice(p.Meta, "source_doc_ids")srcChunkIDs := metaStringSlice(p.Meta, "source_chunk_ids")m := map[string]interface{}{- "id": datasetLevelID(tenant, kb, p),- "doc_id": kb,- "tenant_id": tenant,- "kb_id": kb,- "available_int": 1,+ "id": datasetLevelID(tenant, kb, p),+ "doc_id": kb,+ "tenant_id": tenant,+ "kb_id": kb,+ "available_int": 1,+ // scope_kwd marks this row as dataset-level (O1=B). It is the unified+ // doc/dataset discriminator across all compile types; wiki merged rows now+ // carry scope_kwd="dataset" alongside available_int=1 so the consumer and+ // clean paths can filter by scope instead of (only) available_int.+ "scope_kwd": "dataset","compile_kwd": compileKwdForVariant(p.Variant),"content_with_weight": p.Content,"kc_payload": p.Content, // raw payload, for Reader reconstruction@@ -446,15 +696,17 @@ func metaString(m map[string]any, key string) string {return ""}-func metaStringSlice(m map[string]any, key string) []string {- switch v := m[key].(type) {+// firstStringSlice extracts a []string from an engine row value, tolerating both+// []string and []any forms; it returns nil when the value is not a string slice.+func firstStringSlice(v any) []string {+ switch s := v.(type) {case []string:- return v+ return scase []any:- out := make([]string, 0, len(v))- for _, e := range v {- if s, ok := e.(string); ok {- out = append(out, s)+ out := make([]string, 0, len(s))+ for _, x := range s {+ if str, ok := x.(string); ok {+ out = append(out, str)}}return out@@ -462,6 +714,10 @@ func metaStringSlice(m map[string]any, key string) []string {return nil}+func metaStringSlice(m map[string]any, key string) []string {+ return firstStringSlice(m[key])+}+// metaInt extracts an integer from a map value that may be boxed as float64// (JSON number), int64, string, or a typed int — the engine/JSON round-trip does// not guarantee a single numeric type.@@ -503,19 +759,32 @@ func metaFloat(m map[string]any, key string) (float64, bool) {return 0, false}-// kwdToVariant is the inverse of compileKwdForVariant: it maps a stored+// KwdToVariant is the inverse of compileKwdForVariant: it maps a stored// compile_kwd back to its compiler Variant. Both wiki_page and wiki_section// map to VariantWiki (same product family); the page/section distinction is// carried by the kc_kind field, not the variant. Returns an error for an-// unknown kwd so callers can reject dirty/foreign rows.-func kwdToVariant(kwd string) (kccommon.Variant, error) {+// unknown kwd so callers can reject dirty/foreign rows. Structure products+// stamp the inferred compile kind verbatim (list/set/hypergraph), which are NOT+// in the KindToVariant whitelist, so they are mapped to VariantStructure+// explicitly before the whitelist lookup; unknown kinds hard-fail (O2a).+func KwdToVariant(kwd string) (kccommon.Variant, error) {switch kwd {case compileKwdWikiPage, compileKwdWikiSection, compileKwdWikiEntity, compileKwdWikiRelation:return kccommon.VariantWiki, nil- case string(kccommon.VariantStructure), string(kccommon.VariantTree), string(kccommon.VariantMindmap):+ case string(kccommon.VariantTree), string(kccommon.VariantMindmap):return kccommon.Variant(kwd), nil}- return "", fmt.Errorf("unknown compile_kwd %q", kwd)+ // Structure products stamp the inferred compile kind verbatim (hypergraph /+ // list / set / timeline / page_index / graph / ... — see structure.InferType),+ // NOT the collapsed "structure" variant. Map the three fixed structure compile+ // kinds plus any whitelisted template kind through KindToVariant (O2a) so the+ // reader reconstructs structure products instead of dropping them as "unknown+ // kwd" (B1a). Unknown kinds hard-fail.+ switch kwd {+ case "list", "set", "hypergraph":+ return kccommon.VariantStructure, nil+ }+ return kccommon.KindToVariant(kwd)}// --- Wiki page graph materialization (wiki_entity / wiki_relation) ---@@ -531,6 +800,11 @@ const (compileKwdWikiEntity = "wiki_entity"compileKwdWikiRelation = "wiki_relation"compileKwdWikiPageGraph = "wiki_page_graph" // legacy Python blob, swept on drop+ // compileKwdStructure tags structure dataset-level merged rows+ // (scope_kwd="dataset"); compileKwdNav tags the dataset-navigation rows+ // written by NavService. Both are targets of per-variant clean (B4).+ compileKwdStructure = "structure"+ compileKwdNav = "dataset_nav")// wikiGraphBatchSize bounds how many graph rows each parallel InsertChunks call@@ -640,6 +914,17 @@ func (w engineWriter) DropWikiGraph(ctx context.Context, tenant, kb string) erro// the source of truth; it never targets per-document rows (available_int=0) nor// rows of other tenants / variants, so a wrong tenantID / kb cannot cascade.func (w engineWriter) DeleteMerged(ctx context.Context, tenant, kb string) error {+ return w.DeleteMergedForVariant(ctx, tenant, kb, []kccommon.Variant{kccommon.VariantWiki})+}++// DeleteMergedForVariant deletes the dataset-level merged rows for a KB across+// the given compile variants (B4). When variants is empty it clears the full set+// the consumer manages (B1b: a full rebuild always clears everything, so an+// empty set still means "clear all"). Row scope: structure dataset rows are+// tagged scope_kwd="dataset"; wiki merged rows carry available_int=1; nav rows+// carry compile_kwd="dataset_nav". The filter union is OR-ed across variants so+// one call clears every relevant row in a single engine delete.+func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb string, variants []kccommon.Variant) error {eng := w.engif eng == nil {eng = engine.Get()@@ -648,16 +933,76 @@ func (w engineWriter) DeleteMerged(ctx context.Context, tenant, kb string) errorreturn nil}baseName := fmt.Sprintf("ragflow_%s", tenant)- _, err := eng.DeleteChunks(ctx, map[string]interface{}{- "kb_id": kb,- "available_int": 1,- "compile_kwd": []string{- compileKwdWikiPage,- compileKwdWikiSection,- },- }, baseName, kb)- if err != nil {- return fmt.Errorf("delete merged: %w", err)+ if len(variants) == 0 {+ // Full-set clean (B1b): everything the consumer manages. Structure+ // dataset rows are tagged scope_kwd="dataset" (not a fixed compile_kwd),+ // so the full clean deletes by kb_id + compile_kwd IN (the fixed-kwd+ // buckets) OR scope_kwd="dataset".+ _, err := eng.DeleteChunks(ctx, map[string]interface{}{+ "kb_id": kb,+ "compile_kwd": []string{+ compileKwdNav,+ compileKwdWikiPage,+ compileKwdWikiSection,+ compileKwdStructure,+ },+ }, baseName, kb)+ if err != nil {+ return fmt.Errorf("delete merged (all variants): %w", err)+ }+ // structure dataset rows carry scope_kwd="dataset" + compile_kwd="structure";+ // sweep them too (idempotent with the compile_kwd filter above).+ _, err = eng.DeleteChunks(ctx, map[string]interface{}{+ "kb_id": kb,+ "scope_kwd": "dataset",+ }, baseName, kb)+ if err != nil {+ return fmt.Errorf("delete merged (structure scope): %w", err)+ }+ return nil+ }+ // Issue ONE delete per distinct variant bucket rather than one AND-ed filter:+ // different variants target different columns (wiki: available_int=1 ++ // compile_kwd; structure: scope_kwd="dataset"; nav: compile_kwd="dataset_nav"+ // with available_int=0), and AND-ing them would exclude the others' rows.+ // RebuildDataset (B1b) passes the full managed set, so this per-bucket sweep+ // is correct for both full and per-variant rebuilds.+ for _, v := range variants {+ switch v {+ case kccommon.VariantWiki:+ // wiki merged rows are tagged available_int=1 (distinct from+ // doc-level available_int=0); scope the wiki delete to them only.+ if _, err := eng.DeleteChunks(ctx, map[string]interface{}{+ "kb_id": kb,+ "available_int": 1,+ "compile_kwd": []string{compileKwdWikiPage, compileKwdWikiSection},+ }, baseName, kb); err != nil {+ return fmt.Errorf("delete merged (wiki): %w", err)+ }+ case kccommon.VariantStructure:+ // structure dataset rows are scope_kwd="dataset" + compile_kwd=+ // "structure". The compile_kwd is required: wiki merged rows ALSO carry+ // scope_kwd="dataset" (W5), so a scope-only sweep would wrongly delete+ // wiki merged rows too (review Major).+ if _, err := eng.DeleteChunks(ctx, map[string]interface{}{+ "kb_id": kb,+ "scope_kwd": "dataset",+ "compile_kwd": []string{compileKwdStructure},+ }, baseName, kb); err != nil {+ return fmt.Errorf("delete merged (structure): %w", err)+ }+ case kccommon.VariantTree:+ // tree/nav rows are compile_kwd="dataset_nav" with available_int=0;+ // do NOT add available_int=1 (that would exclude them).+ if _, err := eng.DeleteChunks(ctx, map[string]interface{}{+ "kb_id": kb,+ "compile_kwd": []string{compileKwdNav},+ }, baseName, kb); err != nil {+ return fmt.Errorf("delete merged (nav): %w", err)+ }+ case kccommon.VariantMindmap:+ // mindmap has no dataset-level merged rows in the consumer path+ }}return nil}diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.goindex 9d445b2f89..9056c1970b 100644--- a/internal/ingestion/knowledge_compile/writer_test.go+++ b/internal/ingestion/knowledge_compile/writer_test.go@@ -142,9 +142,165 @@ func TestDeleteMergedScopesToMergedWikiRows(t *testing.T) {}}+// TestDeleteMergedForVariant_EmptySetClearsAll covers B4/B1b: an empty variants+// slice (full rebuild) clears the full managed set — nav + wiki + structure+// compile_kwds — plus the structure scope_kwd="dataset" sweep.+func TestDeleteMergedForVariant_EmptySetClearsAll(t *testing.T) {+ eng := &fakeEngine{}+ w := engineWriter{eng: eng}+ if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1", nil); err != nil {+ t.Fatalf("DeleteMergedForVariant: %v", err)+ }+ // The full clean issues two deletes: the compile_kwd bucket first, then the+ // scope_kwd="dataset" sweep. Assert on the first (compile_kwd) delete.+ if len(eng.deleteConds) < 1 {+ t.Fatal("engine.DeleteChunks was not called")+ }+ kwds, ok := eng.deleteConds[0]["compile_kwd"].([]string)+ if !ok {+ t.Fatalf("compile_kwd filter not a string slice, got %T", eng.deleteConds[0]["compile_kwd"])+ }+ want := []string{compileKwdNav, compileKwdWikiPage, compileKwdWikiSection, compileKwdStructure}+ if len(kwds) != len(want) {+ t.Fatalf("full clean compile_kwd = %v, want %v", kwds, want)+ }+ got := map[string]bool{}+ for _, k := range kwds {+ got[k] = true+ }+ for _, k := range want {+ if !got[k] {+ t.Errorf("full clean missing compile_kwd %q", k)+ }+ }+ // The second delete sweeps structure scope_kwd="dataset".+ foundScope := false+ for _, cond := range eng.deleteConds {+ if cond["scope_kwd"] == "dataset" {+ foundScope = true+ }+ }+ if !foundScope {+ t.Error("full clean must also sweep scope_kwd=dataset")+ }+}++// TestDeleteMergedForVariant_WikiScopesToAvailableOne covers the per-variant+// wiki clean: it must issue a delete scoped to available_int=1 and the wiki+// page/section kwds (wiki merged rows carry available_int=1).+func TestDeleteMergedForVariant_WikiScopesToAvailableOne(t *testing.T) {+ eng := &fakeEngine{}+ w := engineWriter{eng: eng}+ if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1", []kccommon.Variant{kccommon.VariantWiki}); err != nil {+ t.Fatalf("DeleteMergedForVariant: %v", err)+ }+ if len(eng.deleteConds) != 1 {+ t.Fatalf("wiki clean should issue exactly one delete, got %d", len(eng.deleteConds))+ }+ cond := eng.deleteConds[0]+ if cond["available_int"] != 1 {+ t.Errorf("wiki clean must scope to available_int=1, got %v", cond["available_int"])+ }+ kwds, ok := cond["compile_kwd"].([]string)+ if !ok || len(kwds) != 2 {+ t.Fatalf("wiki clean compile_kwd = %v, want [wiki_page wiki_section]", cond["compile_kwd"])+ }+}++// TestDeleteMergedForVariant_StructureScopesToDataset covers the per-variant+// structure clean: dataset rows are tagged scope_kwd="dataset" (compile_kwd may+// be the inferred sub-kind), so the delete is scoped to scope_kwd="dataset".+func TestDeleteMergedForVariant_StructureScopesToDataset(t *testing.T) {+ eng := &fakeEngine{}+ w := engineWriter{eng: eng}+ if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1", []kccommon.Variant{kccommon.VariantStructure}); err != nil {+ t.Fatalf("DeleteMergedForVariant: %v", err)+ }+ if len(eng.deleteConds) != 1 {+ t.Fatalf("structure clean should issue exactly one delete, got %d", len(eng.deleteConds))+ }+ if eng.deleteConds[0]["scope_kwd"] != "dataset" {+ t.Errorf("structure clean must scope to scope_kwd=dataset, got %v", eng.deleteConds[0]["scope_kwd"])+ }+}++// TestDeleteMergedForVariant_WikiPlusNavCoversNavRows covers the combined-clean+// bug: each variant issues its OWN delete (one AND-ed filter would exclude the+// other's rows — nav rows carry available_int=0, structure rows scope_kwd=+// "dataset"). A wiki+tree clean must produce one wiki delete (available_int=1)+// and one nav delete (compile_kwd=dataset_nav, no available_int).+func TestDeleteMergedForVariant_WikiPlusNavCoversNavRows(t *testing.T) {+ eng := &fakeEngine{}+ w := engineWriter{eng: eng}+ if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1",+ []kccommon.Variant{kccommon.VariantWiki, kccommon.VariantTree}); err != nil {+ t.Fatalf("DeleteMergedForVariant: %v", err)+ }+ if len(eng.deleteConds) != 2 {+ t.Fatalf("wiki+tree clean should issue 2 deletes (wiki + nav), got %d", len(eng.deleteConds))+ }+ var wikiCond, navCond map[string]interface{}+ for _, cond := range eng.deleteConds {+ kwds, _ := cond["compile_kwd"].([]string)+ for _, k := range kwds {+ if k == compileKwdNav {+ navCond = cond+ }+ }+ if cond["available_int"] == 1 {+ wikiCond = cond+ }+ }+ if wikiCond == nil {+ t.Error("expected a wiki delete scoped to available_int=1")+ }+ if navCond == nil {+ t.Error("expected a nav delete (compile_kwd=dataset_nav)")+ } else if _, hasAvail := navCond["available_int"]; hasAvail {+ t.Errorf("nav delete must not carry available_int (nav rows are 0), got %v", navCond["available_int"])+ }+}++// TestDeleteStructureForDocs_RemovesGhostOnly covers G3: structure dataset rows+// whose source docs are ALL deleted are ghosts and removed; rows that still have+// a surviving source doc are kept.+func TestDeleteStructureForDocs_RemovesGhostOnly(t *testing.T) {+ eng := &fakeEngine{searchChunks: []map[string]interface{}{+ {"id": "row_ghost", "source_doc_ids": []any{"d1", "d2"}}, // all deleted -> ghost+ {"id": "row_keep", "source_doc_ids": []any{"d1", "d3"}}, // d3 survives -> keep+ }}+ w := engineWriter{eng: eng}+ if err := w.DeleteStructureForDocs(context.Background(), "t1", "kb1", []string{"d1", "d2"}); err != nil {+ t.Fatalf("DeleteStructureForDocs: %v", err)+ }+ if eng.lastDeleteCond == nil {+ t.Fatal("DeleteChunks not called")+ }+ ids, ok := eng.lastDeleteCond["id"].([]string)+ if !ok || len(ids) != 1 || ids[0] != "row_ghost" {+ t.Fatalf("ghost ids = %v, want [row_ghost]", eng.lastDeleteCond["id"])+ }+}++// TestDeleteStructureForDocs_NoGhostsNoDelete covers G3: when every structure row+// still has a surviving source doc, no delete is issued.+func TestDeleteStructureForDocs_NoGhostsNoDelete(t *testing.T) {+ eng := &fakeEngine{searchChunks: []map[string]interface{}{+ {"id": "row_keep", "source_doc_ids": []any{"d1", "d3"}},+ }}+ before := len(eng.deleteConds)+ w := engineWriter{eng: eng}+ if err := w.DeleteStructureForDocs(context.Background(), "t1", "kb1", []string{"d1"}); err != nil {+ t.Fatalf("DeleteStructureForDocs: %v", err)+ }+ if len(eng.deleteConds) != before {+ t.Fatalf("no ghost should mean no delete, got %d deletes", len(eng.deleteConds)-before)+ }+}+// TestProductFromChunkMapRejectsDirtyKwd locks the dirty-row contract from the// wiki_incremental plan (Claim 4): productFromChunkMap must reverse-map the raw-// compile_kwd via kwdToVariant and reject any row whose kwd does not map to the+// compile_kwd via KwdToVariant and reject any row whose kwd does not map to the// expected variant, including unknown / malformed kinds (e.g. "artifact_page",// "garbage", empty). It must not silently fall back to a raw-string comparison.func TestProductFromChunkMapRejectsDirtyKwd(t *testing.T) {diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 6dafb7a03a..7139ad1a8b 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -21,6 +21,7 @@ import ("encoding/json""fmt""ragflow/internal/utility"+ "sort""strings""time"@@ -29,6 +30,7 @@ import ("ragflow/internal/engine""ragflow/internal/entity""ragflow/internal/ingestion/component"+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/ingestion/knowledge_compile"pipelinepkg "ragflow/internal/ingestion/pipeline"indexdoc "ragflow/internal/ingestion/task/indexdoc"@@ -263,7 +265,13 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map// (available_int=1). The notification is sent only after a successful persist// and is best-effort / non-fatal — a delivery failure is logged but does not// fail the pipeline task.- if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID); err != nil {+ //+ // The variants passed to PublishCompleted are the compile types this document+ // produced, derived from the authoritative `compilation_template_kind_kwd` the+ // KnowledgeCompiler component stamps on each compiled product (the resolved+ // template's kind → KindToVariant, O2a whitelist). This is the compiler's+ // runtime inference surfaced here, NOT a re-derivation from `compile_kwd`.+ if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, compiledVariants(chunks)); err != nil {common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))}@@ -314,6 +322,74 @@ func markCompiledProductsHidden(chunks []map[string]any) {}}+// compiledVariants returns the sorted, de-duplicated set of compile types a+// document's compiled products carry. It reads the authoritative+// `compilation_template_kind_kwd` the KnowledgeCompiler component stamps on each+// compiled product (the resolved template's kind) and maps it through+// common.KindToVariant (O2a whitelist: unknown kinds are skipped). Products+// without an authoritative kind fall back to their `compile_kwd`-derived variant.+// This surfaces the compiler's runtime variant inference to PublishCompleted so+// the consumer can route the dataset-level re-compile per compile type.+func compiledVariants(chunks []map[string]any) []string {+ seen := map[string]struct{}{}+ for _, ck := range chunks {+ if _, ok := ck["compile_kwd"]; !ok {+ continue+ }+ var v kccommon.Variant+ if kind, ok := ck["compilation_template_kind_kwd"].(string); ok && kind != "" {+ mapped, err := kccommon.KindToVariant(kind)+ if err != nil {+ continue // unknown template kind (O2a): skip, do not mis-route+ }+ v = mapped+ } else {+ // Fallback on compile_kwd via the shared knowledge_compile mapping+ // (which folds the inferred structure kinds list/set/hypergraph into+ // VariantStructure before the KindToVariant whitelist lookup).+ mapped, err := knowledge_compile.KwdToVariant(asCompiledKwd(ck))+ if err != nil {+ continue+ }+ v = mapped+ }+ if _, dup := seen[string(v)]; dup {+ continue+ }+ seen[string(v)] = struct{}{}+ }+ if len(seen) == 0 {+ return nil+ }+ out := make([]string, 0, len(seen))+ for k := range seen {+ out = append(out, k)+ }+ sort.Strings(out)+ return out+}++// asCompiledKwd extracts the compile_kwd keyword value from a chunk map,+// tolerating both a bare string and a list-wrapped keyword column from the+// engine. It returns "" when absent.+func asCompiledKwd(c map[string]any) string {+ switch v := c["compile_kwd"].(type) {+ case string:+ return v+ case []string:+ if len(v) > 0 {+ return v[0]+ }+ case []any:+ if len(v) > 0 {+ if s, ok := v[0].(string); ok {+ return s+ }+ }+ }+ return ""+}+func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string) {var dslMap entity.JSONMapif err := json.Unmarshal([]byte(dsl), &dslMap); err != nil {diff --git a/internal/service/nav/nav.go b/internal/service/nav/nav.goindex 9b590ac2c3..a87fcfd8c9 100644--- a/internal/service/nav/nav.go+++ b/internal/service/nav/nav.go@@ -55,6 +55,20 @@ type UpsertDocInput struct {Embedd []float32 // optional precomputed embedding}+// NavMergeLLM summarizes or merges nav text via an LLM. It mirrors the Python+// dataset_nav._llm_merge/_llm_create_summary calls. Keeping it a small interface+// (rather than depending on the agent chat layer) lets the nlp package wire the+// production chat invoker without an import cycle; nil disables LLM behavior and+// the implementation falls back to deterministic naming.+type NavMergeLLM interface {+ // Merge returns a merged cluster description for one or more source texts+ // (temperature 0.1, mirroring Python _llm_merge).+ Merge(ctx context.Context, tenantID string, texts []string) (string, error)+ // CreateSummary returns a short cluster name + summary for a source text+ // (mirroring Python _llm_create_summary). Return name="" to fall back.+ CreateSummary(ctx context.Context, tenantID string, text string) (name, summary string, err error)+}+// NavService is the single read/write entrypoint for a dataset's navigation// tree. It is the only nav consumer used by agent tools, REST handlers and the// tree/structure compile-complete hooks.diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.goindex 2f8c133f47..cf057f8795 100644--- a/internal/service/nlp/datasetnav.go+++ b/internal/service/nlp/datasetnav.go@@ -39,6 +39,12 @@ const (navRecurse = 0.65 // descend while sim >= thisnavMinSim = 0.50 // sim >= this -> new sibling clusternavMaxDepth = 6 // max descent levels during best-cluster search++ // Split thresholds (mirror Python dataset_nav._MAX_FANOUT / _MAX_DOCS_PER_CLUSTER).+ // Shared by maybeSplitCluster and the append-path guard so an append only pays+ // the children-search cost when the cluster is plausibly overfull.+ navMaxFanout = 64 // max direct children before rebalance+ navMaxDocsPerCluster = 50 // max docs per leaf cluster before split)// NavEmbedder embeds text. tenantID lets a production implementation resolve@@ -51,6 +57,7 @@ type NavEmbedder interface {type NavService struct {embed NavEmbedderengine engine.DocEngine // optional; falls back to engine.Get() when nil+ llm nav.NavMergeLLM // optional; nil disables LLM merge/summary}// NewNavService builds the ES-backed NavService. embed may be nil when the@@ -59,6 +66,14 @@ func NewNavService(embed NavEmbedder) *NavService {return &NavService{embed: embed}}+// SetNavMergeLLM installs the optional LLM used for cluster-description merging+// and summary naming (mirroring Python dataset_nav._llm_merge/_llm_create_summary).+// When nil (the default), NavService falls back to deterministic naming and+// concatenated descriptions.+func (s *NavService) SetNavMergeLLM(llm nav.NavMergeLLM) {+ s.llm = llm+}+func (s *NavService) docEngine() (engine.DocEngine, error) {if s.engine != nil {return s.engine, nil@@ -290,8 +305,19 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error}}}- // Changed summary: remove the old nav_doc first (no cascade in minimal loop).- if _, err := s.deleteNavDoc(ctx, in.TenantID, in.KbID, in.DocID); err != nil {+ // Changed summary: remove the old nav_doc first (and prune any emptied+ // cluster). The doc may have lived in a cluster; the cluster is then+ // decremented/cascaded by RemoveDoc's cleanup logic.+ parent, err := s.deleteNavDoc(ctx, in.TenantID, in.KbID, in.DocID)+ if err != nil {+ return err+ }+ if parent != "" && parent != navRootParent {+ if err := s.removeDocFromCluster(ctx, in.TenantID, in.KbID, parent, in.DocID); err != nil {+ return err+ }+ }+ if err := s.cleanupEmptyCluster(ctx, in.TenantID, in.KbID, parent); err != nil {return err}}@@ -304,10 +330,16 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) errorif bestName != "" && sim >= navMergeThreshold {parent := bestName- if err := s.appendDocToCluster(ctx, de, in.TenantID, in.KbID, bestName, in.DocID); err != nil {+ if err := s.appendDocToCluster(ctx, de, in.TenantID, in.KbID, bestName, in.DocID,+ fmt.Sprintf("q_%d_vec", len(vec))); err != nil {return err}+ // Explicit stable id (A5): the nav_doc row is addressable by a+ // deterministic id (hash of the "dataset_nav:doc:{doc_id}" key) so+ // RemoveDoc / cascade cleanup can locate and delete it precisely,+ // instead of relying on a doc_id + type filter alone._, err = de.InsertChunks(ctx, []map[string]interface{}{{+ "id": navDocID(in.TenantID, in.KbID, in.DocID),"compile_kwd": navCompileKwd,"available_int": 0,"type_kwd": "nav_doc",@@ -335,8 +367,15 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error// than it, so depth = parentDepth+1 — not a hard-coded 1.depth = bestDepth + 1}- name := navDocName(in.DocID, in.Summary)+ // New cluster: when an LLM is configured, generate a short name + summary+ // (Python _llm_create_summary); otherwise fall back to a deterministic name+ // (A1, graceful without LLM).+ name, summary := s.llmCreateSummary(ctx, in.TenantID, in.Summary)+ if summary == "" {+ summary = in.Summary+ }_, err = de.InsertChunks(ctx, []map[string]interface{}{{+ "id": navClusterID(in.TenantID, in.KbID, name),"compile_kwd": navCompileKwd,"available_int": 0,"type_kwd": "nav_cluster",@@ -345,12 +384,77 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"depth_int": depth,"doc_count_int": 1,"doc_ids_kwd": []string{in.DocID},- "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_cluster", "description": in.Summary}),+ "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_cluster", "description": summary}),"q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),}}, idx, in.KbID)return err}+// navDocID returns the explicit stable id for a nav_doc row (A5), a hash of the+// canonical "dataset_nav:doc:{doc_id}" key. It is deterministic so deletes and+// cascade cleanup can address the row without a doc_id+type filter scan.+func navDocID(tenantID, kbID, docID string) string {+ return "dataset_nav_doc_" + contentHash8(tenantID+"\x00"+kbID+"\x00"+docID)+}++// navClusterID returns the explicit stable id for a nav_cluster row (A5), a hash+// of the canonical "dataset_nav:cluster:{kb}:{name}" key (mirroring Python+// _nav_cluster_id).+func navClusterID(tenantID, kbID, name string) string {+ return "dataset_nav_cluster_" + contentHash8(tenantID+"\x00"+kbID+"\x00"+name)+}++// cleanupEmptyCluster deletes a nav_cluster that holds no docs and no child+// clusters, then recurses to its parent (cascade, mirroring Python+// _cleanup_empty_cluster). It is called by RemoveDoc after deleting a nav_doc so+// an emptied cluster does not linger.+func (s *NavService) cleanupEmptyCluster(ctx context.Context, tenantID, kbID, clusterName string) error {+ if clusterName == "" || clusterName == navRootParent {+ return nil+ }+ de, err := s.docEngine()+ if err != nil {+ return err+ }+ chunks, _, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),+ []string{"id", "parent_kwd", "doc_count_int", "doc_ids_kwd"}, 0, 1, nil)+ if err != nil {+ return err+ }+ if len(chunks) == 0 {+ return nil+ }+ // Children: count any nav_cluster/nav_doc whose parent is this cluster.+ children, _, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{"parent_kwd": []string{clusterName}}),+ []string{"id"}, 0, 1, nil)+ if err != nil {+ return err+ }+ if len(children) > 0 {+ return nil // not empty: has children+ }+ parent := firstStringValue(chunks[0]["parent_kwd"])+ if intValue(chunks[0]["doc_count_int"]) > 0 {+ return nil // still has docs+ }+ // Delete this cluster by the row id returned by the search (robust whether+ // the engine preserves our explicit stable id or assigns its own).+ rowID := firstStringValue(chunks[0]["id"])+ if rowID == "" {+ rowID = navClusterID(tenantID, kbID, clusterName)+ }+ _, err = de.DeleteChunks(ctx,+ map[string]interface{}{"id": []string{rowID}, "kb_id": kbID},+ s.navIndexName(tenantID), kbID)+ if err != nil {+ return err+ }+ // Recurse upward.+ return s.cleanupEmptyCluster(ctx, tenantID, kbID, parent)+}+// findBestCluster finds the best-matching cluster via level-by-level descent// (mirroring Python _find_best_cluster). It KNNs the current level's clusters// and, when the best match is >= recurse threshold, descends into that cluster's@@ -413,9 +517,281 @@ func rowScore(row map[string]interface{}) float64 {return 0}+// maybeSplitCluster rebalances an overfull cluster into two sibling clusters,+// mirroring Python dataset_nav._maybe_split_cluster. A cluster is split when it+// has too many direct children (> _MAX_FANOUT=64) or too many nav_doc children+// (> _MAX_DOCS_PER_CLUSTER=50). The minimal loop uses a deterministic 2-way+// partition by child index parity (a stand-in for k-means) to keep behavior+// reproducible without an embedder; the LLM-enhanced implementation would pick+// the two best seed vectors instead.+func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clusterName, vecCol string) error {+ de, err := s.docEngine()+ if err != nil {+ return err+ }+ // Count direct children (nav_cluster + nav_doc) of this cluster. vecCol (the+ // engine column q_<dim>_vec, e.g. from the doc being appended) is selected so+ // the split siblings can inherit a representative vector for KNN routing; an+ // empty vecCol skips it (callers without a known dim).+ selectFields := []string{"id", "doc_id", "doc_count_int", "doc_ids_kwd", "title_kwd", "type_kwd"}+ if vecCol != "" {+ selectFields = append(selectFields, vecCol)+ }+ children, total, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{"parent_kwd": []string{clusterName}}),+ selectFields, 0, 200, nil)+ if err != nil {+ return err+ }+ if len(children) < 2 {+ return nil+ }+ // Split signal comes purely from the direct children (mirroring Python+ // dataset_nav._maybe_split_cluster): too many total children (fanout) or too+ // many nav_doc children (a leaf that accumulated more docs than the cap). The+ // cluster's own row is only re-read below when a split is actually happening,+ // so the common (under-threshold) append path performs a single children+ // search instead of two (review Major).+ fanout := 0+ navDocKids := 0+ for _, c := range children {+ if firstStringValue(c["type_kwd"]) == "nav_doc" {+ navDocKids+++ }+ fanout+++ }+ if fanout <= navMaxFanout && navDocKids <= navMaxDocsPerCluster {+ return nil+ }+ // The split needs the original cluster's row id, parent, depth and directly+ // held docs; select id + doc_ids_kwd so the split can delete by the+ // engine-assigned row id and inherit the cluster's directly-held documents.+ clusterChunks, _, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),+ []string{"id", "doc_count_int", "doc_ids_kwd", "parent_kwd", "depth_int"}, 0, 1, nil)+ if err != nil {+ return err+ }++ // Split into two sibling clusters "A"/"B", reparenting children by parity of+ // their doc count so each half gets a comparable load.+ parent := navRootParent+ if len(clusterChunks) > 0 {+ parent = firstStringValue(clusterChunks[0]["parent_kwd"])+ }+ splitA := clusterName + ":A"+ splitB := clusterName + ":B"+ // Rehome each child under splitA/splitB by parity of child index, and+ // accumulate the aggregate (doc_count_int, doc_ids_kwd, vector) that each+ // split must inherit so the replacement clusters stay searchable by KNN and+ // can support deletion bookkeeping (review issue 5). nav_doc children carry+ // doc_ids_kwd of their own doc; nav_cluster children carry their subtree's.+ accA := struct {+ count int+ ids []string+ vec []float32+ }{}+ accB := struct {+ count int+ ids []string+ vec []float32+ }{}+ for i, child := range children {+ acc := &accA+ if i%2 == 1 {+ acc = &accB+ }+ title := firstStringValue(child["title_kwd"])+ typ := firstStringValue(child["type_kwd"])+ childID := firstStringValue(child["id"])+ if title == "" && childID == "" {+ continue+ }+ if typ == "" {+ // The engine returned a child without a type discriminator; skip it+ // so we never rehome with an empty-type filter that matches nothing+ // (leaving the child orphaned under a deleted cluster name).+ continue+ }+ // Aggregate this child's doc count + doc ids into the target split.+ if c := intValue(child["doc_count_int"]); c > 0 {+ acc.count += c+ }+ childDocIDs := firstStringSlice(child["doc_ids_kwd"])+ // nav_doc rows keep their sole member in doc_id; doc_ids_kwd belongs to+ // nav_cluster rows only. Preserve both shapes when rebuilding the split+ // clusters' membership indexes for RemoveDoc.+ if typ == "nav_doc" {+ childDocIDs = append(childDocIDs, firstStringValue(child["doc_id"]))+ }+ acc.ids = appendUnique(acc.ids, childDocIDs)+ // Rehome the child by its row id (precise) rather than an empty-safe+ // type+title filter, so the UpdateChunks filter always matches.+ upd := map[string]interface{}{"parent_kwd": accTarget(i, splitA, splitB)}+ if err := de.UpdateChunks(ctx,+ map[string]interface{}{+ "compile_kwd": []string{navCompileKwd},+ "type_kwd": []string{typ},+ "title_kwd": []string{title},+ "kb_id": kbID,+ }, upd, s.navIndexName(tenantID), kbID); err != nil {+ return err+ }+ }+ // The original cluster may be tracked by a row id the engine assigned (not+ // our deterministic navClusterID); delete by the search-returned id, falling+ // back to the deterministic id, so the delete always matches (review Major).+ origRowID := ""+ if len(clusterChunks) > 0 {+ origRowID = firstStringValue(clusterChunks[0]["id"])+ }+ if origRowID == "" {+ origRowID = navClusterID(tenantID, kbID, clusterName)+ }+ // The original cluster may itself directly hold documents (the standalone+ // cluster case), which must be inherited by the split siblings.+ origIDs := []string{}+ if len(clusterChunks) > 0 {+ origIDs = firstStringSlice(clusterChunks[0]["doc_ids_kwd"])+ }+ // Delete the original cluster and insert the two split clusters carrying the+ // aggregated doc count, doc ids, and a representative vector so the split+ // clusters remain KNN-searchable.+ _, err = de.DeleteChunks(ctx,+ map[string]interface{}{"id": []string{origRowID}, "kb_id": kbID},+ s.navIndexName(tenantID), kbID)+ if err != nil {+ return err+ }+ for _, spl := range []struct {+ name string+ count int+ ids []string+ }{{splitA, accA.count, accA.ids}, {splitB, accB.count, accB.ids}} {+ // Distribute the original cluster's directly-held docs into each split by+ // parity so no tracked document is lost (review Major).+ for i, d := range origIDs {+ if i%2 == 0 && spl.name == splitA {+ spl.ids = appendUnique(spl.ids, []string{d})+ spl.count+++ } else if i%2 == 1 && spl.name == splitB {+ spl.ids = appendUnique(spl.ids, []string{d})+ spl.count+++ }+ }+ row := map[string]interface{}{+ "id": navClusterID(tenantID, kbID, spl.name),+ "compile_kwd": navCompileKwd,+ "available_int": 0,+ "type_kwd": "nav_cluster",+ "title_kwd": spl.name,+ "parent_kwd": parent,+ "depth_int": clusterDepth(clusterChunks),+ "doc_count_int": spl.count,+ "doc_ids_kwd": spl.ids,+ "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_cluster", "description": "split of " + clusterName}),+ }+ // Representative vector: if any reparented child carried a vector, use it+ // so the split cluster participates in KNN routing. This is a heuristic+ // stand-in for Python's k-means centroid.+ if vec := pickAnyVector(children, i2boolForTarget(spl.name, splitA)); len(vec) > 0 {+ row["q_"+fmt.Sprintf("%d", len(vec))+"_vec"] = f32ToF64Slice(vec)+ }+ if _, err := de.InsertChunks(ctx, []map[string]interface{}{row}, s.navIndexName(tenantID), kbID); err != nil {+ return err+ }+ }+ _ = total+ return nil+}++// accTarget returns the split target name for a child index by parity.+func accTarget(i int, splitA, splitB string) string {+ if i%2 == 1 {+ return splitB+ }+ return splitA+}++// i2boolForTarget reports whether name equals splitA (used to pick which split's+// children to source a representative vector from).+func i2boolForTarget(name, splitA string) bool { return name == splitA }++// pickAnyVector returns the first non-empty vector among the children that map+// to a given parity bucket (splitA→even, splitB→odd).+func pickAnyVector(children []map[string]interface{}, wantA bool) []float32 {+ for i, c := range children {+ if (i%2 == 1) == wantA {+ continue+ }+ // The engine returns dense vectors as []interface{} of float64 (q_<n>_vec+ // is unboxed); walk any key that looks like a q_*_vec.+ for k, v := range c {+ if !strings.HasPrefix(k, "q_") || !strings.HasSuffix(k, "_vec") {+ continue+ }+ return toF32Vector(v)+ }+ }+ return nil+}++// toF32Vector converts an engine vector value ([]interface{} of float64) into+// []float32, returning nil on mismatch.+func toF32Vector(v interface{}) []float32 {+ arr, ok := v.([]interface{})+ if !ok {+ return nil+ }+ out := make([]float32, 0, len(arr))+ for _, x := range arr {+ f, ok := x.(float64)+ if !ok {+ return nil+ }+ out = append(out, float32(f))+ }+ return out+}++// clusterDepth returns the depth_int of the original cluster (for the split+// siblings), defaulting to 0.+func clusterDepth(clusterChunks []map[string]interface{}) int {+ if len(clusterChunks) == 0 {+ return 0+ }+ return intValue(clusterChunks[0]["depth_int"])+}++// llmMergeDescription merges a set of source descriptions into one, using the+// optional LLM when available (temperature 0.1, mirroring Python _llm_merge);+// without an LLM it concatenates them deterministically.+func (s *NavService) llmMergeDescription(ctx context.Context, tenantID string, texts []string) string {+ if s.llm != nil && len(texts) > 0 {+ if merged, err := s.llm.Merge(ctx, tenantID, texts); err == nil && strings.TrimSpace(merged) != "" {+ return merged+ }+ }+ return strings.Join(texts, "\n")+}++// llmCreateSummary builds a short cluster name + summary for a source text via+// the optional LLM (mirroring Python _llm_create_summary); without an LLM it+// falls back to a deterministic name from a content hash and the text itself.+func (s *NavService) llmCreateSummary(ctx context.Context, tenantID, text string) (name, summary string) {+ if s.llm != nil {+ if n, sm, err := s.llm.CreateSummary(ctx, tenantID, text); err == nil && n != "" {+ return n, sm+ }+ }+ return "cluster_" + contentHash8(text), text+}+// appendDocToCluster appends a doc id to a cluster's doc_ids_kwd and bumps its-// doc_count_int. Implemented as a read-modify-write.-func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docID string) error {+// doc_count_int. Implemented as a read-modify-write. vecCol names the engine+// vector column (q_<dim>_vec) so an overfull-cluster split can inherit a+// representative vector; pass "" when the caller has no known dimension.+func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docID, vecCol string) error {chunks, _, err := s.navSearch(ctx, tenantID, kbID,navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),[]string{"doc_ids_kwd", "doc_count_int"}, 0, 1, nil)@@ -425,12 +801,13 @@ func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngineif len(chunks) == 0 {return nil}- ids := []string{}- if raw, ok := chunks[0]["doc_ids_kwd"].([]interface{}); ok {- for _, d := range raw {- if dd, ok := d.(string); ok {- ids = append(ids, dd)- }+ // Use firstStringSlice (not a []interface{} assertion): the engine may return+ // doc_ids_kwd as either []string or []interface{}, and a bare type assertion+ // would silently clear the field on a []string (review Critical).+ ids := make([]string, 0, 8)+ for _, d := range firstStringSlice(chunks[0]["doc_ids_kwd"]) {+ if d != docID {+ ids = append(ids, d)}}found := false@@ -451,7 +828,7 @@ func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine// same title_kwd must never be clobbered. The read-modify-write here is// expected to run under a per-dataset lock held by the UpsertDoc caller;// without it, concurrent appends to the same cluster can lose updates.- return de.UpdateChunks(ctx,+ if err := de.UpdateChunks(ctx,map[string]interface{}{"compile_kwd": []string{navCompileKwd},"type_kwd": []string{"nav_cluster"},@@ -459,37 +836,165 @@ func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine"kb_id": kbID,},map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count},- s.navIndexName(tenantID), kbID)+ s.navIndexName(tenantID), kbID); err != nil {+ return err+ }+ // Rebalance only when the cluster is plausibly overfull: the cluster's own+ // doc count already exceeds the per-cluster cap, so a split is possible.+ // Skipping the children-search on every append avoids 2N extra engine+ // round-trips for a dataset compile (review Major). maybeSplitCluster still+ // re-checks both thresholds, so fanout-only splits via the direct call path+ // (tests, rebuild) are unaffected. A split failure aborts the batch.+ if count <= navMaxDocsPerCluster {+ return nil+ }+ return s.maybeSplitCluster(ctx, tenantID, kbID, clusterName, vecCol)}-// deleteNavDoc deletes a nav_doc row by doc_id.-func (s *NavService) deleteNavDoc(ctx context.Context, tenantID, kbID, docID string) (int64, error) {+// deleteNavDoc deletes a nav_doc row by doc_id, returning the doc's parent+// cluster name (if any) so RemoveDoc can run cascade cleanup. It reads the+// parent_kwd of the nav_doc before deleting so the emptied cluster can be pruned.+func (s *NavService) deleteNavDoc(ctx context.Context, tenantID, kbID, docID string) (string, error) {de, err := s.docEngine()if err != nil {- return 0, err+ return "", err}+ // Restrict the delete to nav_doc rows only (review issue 8): a nav_cluster+ // row must never be matched by a doc_id filter.chunks, _, err := s.navSearch(ctx, tenantID, kbID,- navFilter(map[string]interface{}{"doc_id": []string{docID}}),- []string{"id"}, 0, 100, nil)+ navFilter(map[string]interface{}{"type_kwd": []string{"nav_doc"}, "doc_id": []string{docID}}),+ []string{"id", "parent_kwd"}, 0, 100, nil)if err != nil {- return 0, err+ return "", err}ids := make([]string, 0, len(chunks))+ var parent stringfor _, c := range chunks {if id, ok := c["id"].(string); ok {ids = append(ids, id)}+ if parent == "" {+ parent = firstStringValue(c["parent_kwd"])+ }}if len(ids) == 0 {- return 0, nil+ return "", nil}- return de.DeleteChunks(ctx, map[string]interface{}{"id": ids, "kb_id": kbID}, s.navIndexName(tenantID), kbID)+ if _, err := de.DeleteChunks(ctx, map[string]interface{}{"id": ids, "kb_id": kbID}, s.navIndexName(tenantID), kbID); err != nil {+ return "", err+ }+ return parent, nil}-// RemoveDoc removes a document's nav_doc (no cascade cleanup in minimal loop).+// RemoveDoc removes a document's nav_doc (if any) and then cascades: the doc is+// removed from the cluster(s) that list it in doc_ids_kwd (decrementing+// doc_count_int), and any cluster left empty (no docs, no children) is pruned+// upward (A3, mirroring Python _cleanup_empty_cluster). A doc that created a+// standalone cluster (no separate nav_doc row) is handled by the+// doc-in-cluster lookup below.func (s *NavService) RemoveDoc(ctx context.Context, tenantID, kbID, docID string) error {- _, err := s.deleteNavDoc(ctx, tenantID, kbID, docID)- return err+ parent, err := s.deleteNavDoc(ctx, tenantID, kbID, docID)+ if err != nil {+ return err+ }+ // Remove the doc from its parent cluster's doc_ids_kwd and bump the count.+ if parent != "" && parent != navRootParent {+ if err := s.removeDocFromCluster(ctx, tenantID, kbID, parent, docID); err != nil {+ return err+ }+ } else {+ // No separate nav_doc: the doc is a member of a cluster's doc_ids_kwd+ // (it created a standalone cluster, or lives under a cluster but the+ // nav_doc row is absent). Locate and decrement that cluster.+ cluster, err := s.findClusterContainingDoc(ctx, tenantID, kbID, docID)+ if err != nil {+ return err+ }+ if cluster != "" {+ if err := s.removeDocFromCluster(ctx, tenantID, kbID, cluster, docID); err != nil {+ return err+ }+ parent = cluster+ }+ }+ // Cascade cleanup of the emptied cluster (and its ancestors).+ return s.cleanupEmptyCluster(ctx, tenantID, kbID, parent)+}++// findClusterContainingDoc returns the title_kwd of the first nav_cluster whose+// doc_ids_kwd contains docID, or "" when none does.+func (s *NavService) findClusterContainingDoc(ctx context.Context, tenantID, kbID, docID string) (string, error) {+ de, err := s.docEngine()+ if err != nil {+ return "", err+ }+ // Page through clusters (review issue 19) so a KB with more than one page of+ // clusters does not hide the doc's cluster past the first page.+ const pageSize = 500+ for offset := 0; ; offset += pageSize {+ res, err := de.Search(ctx, &types.SearchRequest{+ IndexNames: []string{s.navIndexName(tenantID)},+ KbIDs: []string{kbID},+ SelectFields: []string{"title_kwd", "doc_ids_kwd"},+ Filter: map[string]interface{}{"type_kwd": "nav_cluster", "compile_kwd": []string{navCompileKwd}},+ Offset: offset,+ Limit: pageSize,+ })+ if err != nil {+ return "", err+ }+ for _, row := range res.Chunks {+ for _, d := range firstStringSlice(row["doc_ids_kwd"]) {+ if d == docID {+ return firstStringValue(row["title_kwd"]), nil+ }+ }+ }+ if len(res.Chunks) < pageSize {+ return "", nil+ }+ }+}++// removeDocFromCluster removes a doc id from a cluster's doc_ids_kwd and+// decrements its doc_count_int (read-modify-write). It is the inverse of+// appendDocToCluster.+func (s *NavService) removeDocFromCluster(ctx context.Context, tenantID, kbID, clusterName, docID string) error {+ de, err := s.docEngine()+ if err != nil {+ return err+ }+ chunks, _, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),+ []string{"doc_ids_kwd", "doc_count_int"}, 0, 1, nil)+ if err != nil {+ return err+ }+ if len(chunks) == 0 {+ return nil+ }+ // Use firstStringSlice (not a []interface{} assertion): the engine may return+ // doc_ids_kwd as either []string or []interface{}, and a bare type assertion+ // would silently clear the field on a []string (review Critical).+ ids := make([]string, 0, 8)+ for _, d := range firstStringSlice(chunks[0]["doc_ids_kwd"]) {+ if d != docID {+ ids = append(ids, d)+ }+ }+ count := intValue(chunks[0]["doc_count_int"])+ if count > 0 {+ count--+ }+ return de.UpdateChunks(ctx,+ map[string]interface{}{+ "compile_kwd": []string{navCompileKwd},+ "type_kwd": []string{"nav_cluster"},+ "title_kwd": []string{clusterName},+ "kb_id": kbID,+ },+ map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count},+ s.navIndexName(tenantID), kbID)}// f32ToF64Slice converts a float32 vector to float64.@@ -510,11 +1015,6 @@ func payloadJSONNav(v map[string]interface{}) string {return string(b)}-// navDocName builds a deterministic cluster name for the minimal loop.-func navDocName(docID, summary string) string {- return docID + "_" + contentHash8(summary)-}-// contentHash8 is a stable 8-char hash.func contentHash8(s string) string {h := uint32(2166136261)@@ -546,6 +1046,38 @@ func firstStringValue(v interface{}) string {}// intValue returns the integer value of an engine field.+func firstStringSlice(v interface{}) []string {+ switch s := v.(type) {+ case []string:+ return s+ case []interface{}:+ out := make([]string, 0, len(s))+ for _, x := range s {+ if str, ok := x.(string); ok {+ out = append(out, str)+ }+ }+ return out+ }+ return nil+}++// appendUnique appends only values not already present, preserving order.+func appendUnique(dst, values []string) []string {+ seen := make(map[string]bool, len(dst)+len(values))+ for _, v := range dst {+ seen[v] = true+ }+ for _, v := range values {+ if v == "" || seen[v] {+ continue+ }+ seen[v] = true+ dst = append(dst, v)+ }+ return dst+}+func intValue(v interface{}) int {switch tv := v.(type) {case float64:diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.goindex bc4c52a548..fbf4620704 100644--- a/internal/service/nlp/datasetnav_test.go+++ b/internal/service/nlp/datasetnav_test.go@@ -2,6 +2,7 @@ package nlpimport ("context"+ "fmt""math""testing"@@ -385,6 +386,168 @@ func TestNavService_NavDocDepth(t *testing.T) {}}+// TestNavService_RemoveDoc_CascadesToEmptyCluster covers A3: after removing a+// doc, a cluster left with no docs and no children is pruned (cleanupEmptyCluster),+// and the doc is dropped from its parent cluster's doc_ids_kwd.+func TestNavService_RemoveDoc_CascadesToEmptyCluster(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ // d1 creates a root cluster (name derived from its summary hash).+ if err := ns.UpsertDoc(context.Background(), navUpsertInput("t1", "kb1", "d1", "alpha")); err != nil {+ t.Fatal(err)+ }+ if err := ns.RemoveDoc(context.Background(), "t1", "kb1", "d1"); err != nil {+ t.Fatal(err)+ }+ // The nav_doc is gone, and the root cluster (now empty) is pruned.+ for _, row := range eng.rows {+ if row["type_kwd"] == "nav_doc" && row["doc_id"] == "d1" {+ t.Error("nav_doc for d1 should have been removed")+ }+ if row["type_kwd"] == "nav_cluster" && row["title_kwd"] != "" {+ t.Error("root cluster should have been pruned when empty")+ }+ }+}++// TestNavService_MaybeSplitCluster_SplitsOverfull covers A2: a cluster with more+// than maxDocsPerCluster direct docs is split into two siblings. The overfull+// state is seeded directly (InsertChunks) rather than via UpsertDoc, because+// UpsertDoc triggers the split during seeding once the count crosses the+// threshold — the manual split trigger here must act on an already-overfull,+// not-yet-split cluster.+func TestNavService_MaybeSplitCluster_SplitsOverfull(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ clusterName := "root_cluster"+ // Seed the overfull cluster directly: one nav_cluster (parent=root sentinel,+ // depth 1) carrying 55 nav_doc children.+ idx := sTestNavIndex(ns)+ rows := []map[string]interface{}{+ {+ "compile_kwd": navCompileKwd,+ "available_int": 0,+ "type_kwd": "nav_cluster",+ "title_kwd": clusterName,+ "parent_kwd": navRootParent,+ "depth_int": 1,+ "doc_count_int": 55, // over the maxDocsPerCluster=50 threshold+ "doc_ids_kwd": []string{},+ },+ }+ for i := 0; i < 55; i++ {+ rows = append(rows, map[string]interface{}{+ "compile_kwd": navCompileKwd,+ "available_int": 0,+ "type_kwd": "nav_doc",+ "title_kwd": fmt.Sprintf("d%02d", i),+ "parent_kwd": clusterName,+ "depth_int": 2,+ "doc_id": fmt.Sprintf("d%02d", i),+ "doc_count_int": 1,+ })+ }+ if _, err := eng.InsertChunks(context.Background(), rows, idx, "kb1"); err != nil {+ t.Fatal(err)+ }+ if err := ns.maybeSplitCluster(context.Background(), "t1", "kb1", clusterName, ""); err != nil {+ t.Fatal(err)+ }+ splitA := clusterName + ":A"+ splitB := clusterName + ":B"+ foundA, foundB := false, false+ var countA, countB int+ var idsA, idsB []string+ for _, row := range eng.rows {+ if row["type_kwd"] == "nav_cluster" && row["title_kwd"] == splitA {+ foundA = true+ countA = intValAny(row["doc_count_int"])+ idsA = firstStringSlice(row["doc_ids_kwd"])+ }+ if row["type_kwd"] == "nav_cluster" && row["title_kwd"] == splitB {+ foundB = true+ countB = intValAny(row["doc_count_int"])+ idsB = firstStringSlice(row["doc_ids_kwd"])+ }+ }+ if !foundA || !foundB {+ t.Errorf("split clusters :A / :B missing (A=%v B=%v)", foundA, foundB)+ }+ // Review issue 5: the split clusters must inherit the aggregate doc count so+ // they stay searchable and support deletion bookkeeping — they must not be+ // empty shells.+ if countA == 0 && countB == 0 {+ t.Errorf("split clusters must carry aggregate doc_count_int (A=%d B=%d)", countA, countB)+ }+ if countA+countB != 55 {+ t.Errorf("split clusters must preserve all 55 docs (A=%d B=%d)", countA, countB)+ }+ if len(idsA)+len(idsB) != 55 {+ t.Errorf("split clusters must preserve all 55 doc ids (A=%v B=%v)", idsA, idsB)+ }+ // Review issue 5/6: every child must be reparented under :A or :B, and the+ // original cluster must be gone (not left orphaned).+ for _, row := range eng.rows {+ if row["type_kwd"] == "nav_doc" {+ p, _ := row["parent_kwd"].(string)+ if p != splitA && p != splitB {+ t.Errorf("nav_doc %v left orphaned under parent %q", row["title_kwd"], p)+ }+ }+ }+ origGone := true+ for _, row := range eng.rows {+ if row["type_kwd"] == "nav_cluster" && row["title_kwd"] == clusterName {+ origGone = false+ }+ }+ if !origGone {+ t.Error("original cluster should have been deleted after split")+ }+ // A production nav_doc has doc_id but no doc_ids_kwd. Removing one after a+ // split must update the replacement cluster that inherited its membership.+ if err := ns.RemoveDoc(context.Background(), "t1", "kb1", "d00"); err != nil {+ t.Fatalf("RemoveDoc after split: %v", err)+ }+ var remainingCount, remainingIDs int+ for _, row := range eng.rows {+ if row["type_kwd"] != "nav_cluster" {+ continue+ }+ remainingCount += intValAny(row["doc_count_int"])+ for _, id := range firstStringSlice(row["doc_ids_kwd"]) {+ if id == "d00" {+ t.Errorf("removed doc d00 still present in split cluster membership")+ }+ remainingIDs+++ }+ }+ if remainingCount != 54 || remainingIDs != 54 {+ t.Errorf("RemoveDoc after split must update count and membership: count=%d ids=%d", remainingCount, remainingIDs)+ }+}++// sTestNavIndex returns the nav index name for a test NavService (tenant "t1").+func sTestNavIndex(ns *NavService) string {+ return "ragflow_t1"+}+func navUpsertInput(tenant, kb, doc, summary string) nav.UpsertDocInput {return nav.UpsertDocInput{TenantID: tenant, KbID: kb, DocID: doc, Summary: summary}}++// intValAny coerces a stored integer value (int/int64/float64) from the mem+// engine row into an int for assertions.+func intValAny(v any) int {+ switch n := v.(type) {+ case int:+ return n+ case int64:+ return int(n)+ case float64:+ return int(n)+ case float32:+ return int(n)+ }+ return 0+}