Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Port dataset-level knowledge compile to Go with variant dispatch (#18161)

Zhichang Yuc677e9af365f
diff --git a/internal/ingestion/component/knowledge_compiler/common/types_test.go b/internal/ingestion/component/knowledge_compiler/common/types_test.go
new file mode 100644
index 0000000000..38e3da1742
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/common/types_test.go
@@ -0,0 +1,53 @@
+package common
+
+import "testing"
+
+// TestKindToVariant_WhitelistCoversAllValidKinds covers O2a: every whitelisted
+// template kind maps to a non-empty variant without error.
+func TestKindToVariant_WhitelistCoversAllValidKinds(t *testing.T) {
+ validKinds := []string{
+ "tree", "mind_map", "mindmap", "wiki", "page_index",
+ "session_essence", "session_graph", "timeline",
+ "knowledge_graph", "structure", "knowledgegraph", "graph",
+ }
+ for _, kind := range validKinds {
+ v, err := KindToVariant(kind)
+ if err != nil {
+ t.Errorf("KindToVariant(%q) unexpected error: %v", kind, err)
+ }
+ if v == "" {
+ t.Errorf("KindToVariant(%q) returned empty variant", kind)
+ }
+ }
+}
+
+// TestKindToVariant_UnknownKindHardFails covers O2a: a kind outside the
+// whitelist returns an error (no fallback) so the rebuild/component path cannot
+// silently mis-route an unknown template.
+func TestKindToVariant_UnknownKindHardFails(t *testing.T) {
+ for _, kind := range []string{"garbage", "artifact_page", "unknown_xyz", ""} {
+ if _, err := KindToVariant(kind); err == nil {
+ t.Errorf("KindToVariant(%q) should hard-fail (O2a), got nil error", kind)
+ }
+ }
+}
+
+// TestKindToVariant_CanonicalForms covers O2a aliases: the same template kind
+// in different canonical spellings maps to the same variant.
+func TestKindToVariant_CanonicalForms(t *testing.T) {
+ a, err := KindToVariant("structure")
+ if err != nil {
+ t.Fatalf("structure: %v", err)
+ }
+ b, err := KindToVariant("knowledge_graph")
+ if err != nil {
+ t.Fatalf("knowledge_graph: %v", err)
+ }
+ c, err := KindToVariant("graph")
+ if err != nil {
+ t.Fatalf("graph: %v", err)
+ }
+ if a != b || a != c {
+ t.Errorf("structure/knowledge_graph/graph should map to the same variant, got %q %q %q", a, b, c)
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index a52135dea8..aa0b23357b 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -6,7 +6,6 @@ package knowledge_compiler
import (
"context"
- "encoding/json"
"fmt"
"log"
"sort"
@@ -21,7 +20,6 @@ import (
"ragflow/internal/ingestion/component/knowledge_compiler/tree"
"ragflow/internal/ingestion/component/knowledge_compiler/wiki"
"ragflow/internal/ingestion/component/schema"
- "ragflow/internal/service/nav"
"ragflow/internal/tokenizer"
"go.uber.org/zap"
@@ -135,9 +133,6 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
// template's id and kind. All products are buffered (no streaming sink), so
// the post-run loop below covers every row (M1).
var out common.Outputs
- // navByProducts accumulates every tree/structure by-product job so each is
- // written after the spec loop (not just the last one).
- var navByProducts []navByProduct
for _, spec := range specs {
variant, err := common.KindToVariant(spec.Kind)
if err != nil {
@@ -185,13 +180,6 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
return nil, err
}
- // Accumulate tree/structure by-product jobs so each is written after the
- // spec loop (a multi-spec batch must not drop any spec's products). nav is
- // a derived artifact; its failures are logged and never abort the pipeline.
- if variant == common.VariantTree || variant == common.VariantStructure {
- navByProducts = append(navByProducts, navByProduct{deps: deps, variant: variant, products: o.Products})
- }
-
for i := range o.Products {
if o.Products[i].Meta == nil {
o.Products[i].Meta = map[string]any{}
@@ -204,17 +192,6 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
out.Products = append(out.Products, o.Products...)
}
- // Write each dataset-nav by-product after all specs ran. tree by-product
- // (Python compiler.py:475) and structure/page_index by-product (Python
- // runner.py:189). Failures are logged and never abort the pipeline.
- for _, job := range navByProducts {
- if job.variant == common.VariantTree {
- upsertTreeNav(ctx, job.deps, in.DocID, job.products)
- } else {
- upsertStructureNav(ctx, job.deps, in.DocID, job.products)
- }
- }
-
// Convert the compiled products into chunk-aligned docs (matching
// conf/infinity_mapping.json) and merge them into the upstream input
// chunks. The component stays DB-independent and no longer routes through a
@@ -252,142 +229,6 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
return mergeChunks(inputs, compiled), nil
}
-// navByProduct is one deferred dataset-nav by-product job: the compile variant
-// that produced it and the products to summarize from.
-type navByProduct struct {
- deps common.Deps
- variant common.Variant
- products []common.Product
-}
-
-// upsertTreeNav writes the dataset-nav by-product after tree compilation. It
-// finds the tree root product (Meta kind=root) whose Content is the document
-// summary, and places it into the ES-backed nav tree. Any failure (missing nav
-// service, missing root, embedding/upsert error) is logged and skipped — the
-// nav artifact must never block the compile pipeline.
-func upsertTreeNav(ctx context.Context, deps common.Deps, docID string, products []common.Product) {
- ns := nav.GetNavService()
- if ns == nil {
- log.Printf("knowledge_compiler: datasetnav by-product skipped (NavService not initialized)")
- return
- }
- var summary string
- var vec []float32
- for i := range products {
- if kind, _ := products[i].Meta["kind"].(string); kind == "root" {
- summary = products[i].Content
- vec = products[i].Vector
- break
- }
- }
- if strings.TrimSpace(summary) == "" {
- log.Printf("knowledge_compiler: datasetnav by-product skipped (tree has no root summary)")
- return
- }
- if len(vec) == 0 {
- if deps.Embed == nil {
- log.Printf("knowledge_compiler: datasetnav by-product skipped (no embedder, no precomputed vector)")
- return
- }
- embeddings, err := deps.Embed.Encode(ctx, []string{summary})
- if err != nil {
- log.Printf("knowledge_compiler: datasetnav by-product skipped (embedding failed): %v", err)
- return
- }
- if len(embeddings) == 0 {
- log.Printf("knowledge_compiler: datasetnav by-product skipped (embedding produced no vector)")
- return
- }
- vec = embeddings[0]
- }
- if err := ns.UpsertDoc(ctx, nav.UpsertDocInput{
- TenantID: deps.TenantID,
- KbID: deps.DatasetID,
- DocID: docID,
- Summary: summary,
- Embedd: vec,
- }); err != nil {
- // Log and continue: nav is a derived artifact, never abort compile.
- log.Printf("knowledge_compiler: datasetnav by-product upsert failed (continuing): %v", err)
- }
-}
-
-// upsertStructureNav writes the dataset-nav by-product after structure/page_index
-// compilation (mirroring Python runner.py:189 _upsert_dataset_nav_from_page_index).
-// It finds the graph product (Meta kind=graph), summarizes its entity
-// descriptions into a document-level summary, and places it into the nav tree.
-// Failures are logged and never abort the compile pipeline.
-func upsertStructureNav(ctx context.Context, deps common.Deps, docID string, products []common.Product) {
- ns := nav.GetNavService()
- if ns == nil {
- log.Printf("knowledge_compiler: datasetnav by-product skipped (NavService not initialized)")
- return
- }
- var graph *common.Product
- for i := range products {
- if kind, _ := products[i].Meta["kind"].(string); kind == "graph" {
- graph = &products[i]
- break
- }
- }
- if graph == nil {
- return
- }
- summary := pageIndexSummary(graph.Content)
- if strings.TrimSpace(summary) == "" {
- log.Printf("knowledge_compiler: datasetnav by-product skipped (structure graph has no entity descriptions)")
- return
- }
- // Embed the SUMMARY text, not the graph JSON — the nav doc's vector must
- // represent the summary semantics for the KNN router to match correctly.
- if deps.Embed == nil {
- log.Printf("knowledge_compiler: datasetnav by-product skipped (no embedder)")
- return
- }
- embeddings, err := deps.Embed.Encode(ctx, []string{summary})
- if err != nil || len(embeddings) == 0 {
- log.Printf("knowledge_compiler: datasetnav by-product skipped (structure embedding failed): %v", err)
- return
- }
- vec := embeddings[0]
- if err := ns.UpsertDoc(ctx, nav.UpsertDocInput{
- TenantID: deps.TenantID,
- KbID: deps.DatasetID,
- DocID: docID,
- Summary: summary,
- Embedd: vec,
- }); err != nil {
- log.Printf("knowledge_compiler: datasetnav by-product upsert failed (continuing): %v", err)
- }
-}
-
-// pageIndexSummary concatenates entity descriptions from a structure graph JSON
-// ({"entities": [{"name","description"}, ...]}), producing a document-level
-// summary for dataset navigation.
-func pageIndexSummary(graphJSON string) string {
- var graph struct {
- Entities []struct {
- Name string `json:"name"`
- Description string `json:"description"`
- } `json:"entities"`
- }
- if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {
- return ""
- }
- var b strings.Builder
- for _, e := range graph.Entities {
- desc := strings.Join(strings.Fields(e.Description), " ")
- if desc == "" {
- continue
- }
- if e.Name != "" {
- b.WriteString(e.Name + ": ")
- }
- b.WriteString(desc + "\n")
- }
- return b.String()
-}
-
// resolveTemplateSpecs resolves the configured compilation template spec(s) to
// their TemplateInfo rows. Priority: compilation_template_id >
// compilation_template_group_id. The group path resolves the group to its
@@ -547,6 +388,14 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {
if err := doc.SetExtraValue("compilation_template_kind_kwd", kindOrVariant(p)); err != nil {
return nil, err
}
+ // scope_kwd marks doc/dataset-level rows (B8/O1=B). A doc-level compiled
+ // product is always a per-document (scope="doc") input to the dataset-level
+ // merge; the consumer rewrites dataset-level rows with scope_kwd="dataset".
+ // This is the dataset-level writer-layer discriminator, not a doc-level
+ // compile-internal concern.
+ if err := doc.SetExtraValue("scope_kwd", "doc"); err != nil {
+ return nil, err
+ }
if p.ParentID != "" {
if err := doc.SetExtraValue("parent_kwd", p.ParentID); err != nil {
return nil, err
diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.go
index d9f796e81a..dc055d1bcd 100644
--- a/internal/ingestion/component/knowledge_compiler/component_test.go
+++ b/internal/ingestion/component/knowledge_compiler/component_test.go
@@ -15,7 +15,6 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
- "ragflow/internal/service/nav"
"gorm.io/gorm"
)
@@ -258,124 +257,6 @@ func TestKnowledgeCompiler_Datasetnav_NoVariant(t *testing.T) {
}
}
-// fakeNavSvc records the most recent UpsertDoc call so a test can assert the
-// tree by-product hook feeds the nav service the root document summary.
-type fakeNavSvc struct {
- mu sync.Mutex
- called bool
- summary string
- docID string
- kbID string
-}
-
-func (f *fakeNavSvc) UpsertDoc(_ context.Context, in nav.UpsertDocInput) error {
- f.mu.Lock()
- defer f.mu.Unlock()
- f.called = true
- f.summary = in.Summary
- f.docID = in.DocID
- f.kbID = in.KbID
- return nil
-}
-func (f *fakeNavSvc) RemoveDoc(context.Context, string, string, string) error { return nil }
-func (f *fakeNavSvc) Search(context.Context, string, string, string, []float32, int) ([]nav.NavHit, error) {
- return nil, nil
-}
-func (f *fakeNavSvc) ListClusters(context.Context, string, string, int, int) ([]nav.NavNode, int64, error) {
- return nil, 0, nil
-}
-func (f *fakeNavSvc) ListChildren(context.Context, string, string, string, int, int) ([]nav.NavNode, int64, error) {
- return nil, 0, nil
-}
-
-// TestKnowledgeCompiler_TreeNavByProduct asserts the tree compile-complete hook
-// writes the dataset-nav by-product: after tree.Run produces a root product,
-// upsertTreeNav calls NavService.UpsertDoc with the root summary and the
-// doc/dataset context, and never aborts on failure.
-func TestKnowledgeCompiler_TreeNavByProduct(t *testing.T) {
- fake := &fakeNavSvc{}
- nav.SetNavService(fake)
- t.Cleanup(func() { nav.SetNavService(nil) })
-
- deps := common.Deps{
- TenantID: "t1",
- DatasetID: "kb1",
- Chat: proseChat{},
- Embed: mockEmbedder{dim: 8},
- }
- products := []common.Product{
- {Content: "section one body", Meta: map[string]any{"kind": "summary", "level": 0}},
- {Content: "overall doc theme root summary", Vector: []float32{0.1, 0.2}, Meta: map[string]any{"kind": "root", "level": -1}},
- }
- upsertTreeNav(context.Background(), deps, "d1", products)
-
- fake.mu.Lock()
- defer fake.mu.Unlock()
- if !fake.called {
- t.Fatal("upsertTreeNav did not call NavService.UpsertDoc")
- }
- if fake.docID != "d1" {
- t.Errorf("doc_id = %q, want d1", fake.docID)
- }
- if fake.kbID != "kb1" {
- t.Errorf("kb_id = %q, want kb1", fake.kbID)
- }
- if !strings.Contains(fake.summary, "overall doc theme root summary") {
- t.Errorf("summary = %q, want the tree root summary", fake.summary)
- }
-}
-
-// TestKnowledgeCompiler_TreeNavByProduct_NilServiceDoesNotAbort asserts the
-// by-product hook tolerates a missing NavService without erroring (nav is a
-// derived artifact; its absence must never abort compilation).
-func TestKnowledgeCompiler_TreeNavByProduct_NilServiceDoesNotAbort(t *testing.T) {
- nav.SetNavService(nil)
- t.Cleanup(func() { nav.SetNavService(nil) })
- // Should not panic and must return normally.
- upsertTreeNav(context.Background(), common.Deps{}, "d1",
- []common.Product{{Content: "x", Meta: map[string]any{"kind": "root"}}})
-}
-
-// TestKnowledgeCompiler_StructureNavByProduct asserts the structure/page_index
-// by-product hook summarizes the graph product entities and feeds NavService.
-func TestKnowledgeCompiler_StructureNavByProduct(t *testing.T) {
- fake := &fakeNavSvc{}
- nav.SetNavService(fake)
- t.Cleanup(func() { nav.SetNavService(nil) })
-
- graphJSON := `{"entities":[{"name":"Engine","description":"a propulsion device"},{"name":"Fuel","description":"combustion source"}]}`
- products := []common.Product{
- {Content: graphJSON, Vector: []float32{0.1, 0.2}, Meta: map[string]any{"kind": "graph", "compile_kwd": "page_index"}},
- }
- // The by-product embeds the SUMMARY (not the graph JSON vector), so an
- // embedder must be present.
- upsertStructureNav(context.Background(), common.Deps{TenantID: "t1", DatasetID: "kb1", Embed: mockEmbedder{dim: 8}}, "d1", products)
-
- fake.mu.Lock()
- defer fake.mu.Unlock()
- if !fake.called {
- t.Fatal("upsertStructureNav did not call NavService.UpsertDoc")
- }
- if fake.docID != "d1" {
- t.Errorf("doc_id = %q, want d1", fake.docID)
- }
- if !strings.Contains(fake.summary, "Engine: a propulsion device") {
- t.Errorf("summary = %q, want entity descriptions", fake.summary)
- }
-}
-
-// TestPageIndexSummary asserts entity descriptions are concatenated into a
-// document summary.
-func TestPageIndexSummary(t *testing.T) {
- got := pageIndexSummary(`{"entities":[{"name":"A","description":"one thing"},{"name":"B","description":""}]}`)
- if !strings.Contains(got, "A: one thing") {
- t.Errorf("summary = %q, want entity A", got)
- }
- if strings.Contains(got, "B") {
- t.Errorf("summary should skip empty-description entity, got %q", got)
- }
-}
-
func TestKnowledgeCompiler_Alias_Mindmap(t *testing.T) {
installMockDeps(t)
// "mind_map" is the deprecated alias for "mindmap"; both resolve to the
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index 753b017201..dd1c0b5cd8 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -17,9 +17,11 @@ package knowledge_compile
import (
"context"
+ "encoding/json"
"errors"
"fmt"
"sort"
+ "strings"
"sync"
"time"
@@ -27,6 +29,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/engine"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+ "ragflow/internal/service/nav"
"go.uber.org/zap"
"gorm.io/gorm"
@@ -69,7 +72,7 @@ type Consumer struct {
// satisfied by *mysqlScheduler (and FakeScheduler in tests). Using a narrow
// interface keeps the Claimer surface unchanged.
type rewriteScheduler interface {
- Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error
+ Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error
CancelInflight(ctx context.Context, datasetID, token string) error
}
@@ -306,21 +309,37 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)
// lock while the clear runs, so it can never repopulate cleared state with
// old results.
if err := c.scheduler.WithDatasetLock(ctx, kb, func(_ string) error {
- if derr := c.writer.DeleteMerged(ctx, tenant, kb); derr != nil {
+ // B4/B1b: a full rebuild clears EVERY variant the consumer manages
+ // (wiki merged rows, structure scope_kwd="dataset" rows, and the nav
+ // tree rows) via the fixed full-set clean (empty variants), not just the
+ // wiki rows that DeleteMerged removes — otherwise removed-template ghost
+ // rows survive. DropWikiGraph additionally sweeps the legacy wiki graph.
+ if derr := c.writer.DeleteMergedForVariant(ctx, tenant, kb, nil); derr != nil {
return derr
}
return c.writer.DropWikiGraph(ctx, tenant, kb)
}); err != nil {
- return fmt.Errorf("knowledge_compile: rebuild clear merged+graph: %w", err)
+ return fmt.Errorf("knowledge_compile: rebuild clear all variants + graph: %w", err)
}
- // 4. enumerate every doc and republish.
+ // 4. enumerate every doc and republish. Per B1, the republished events must
+ // carry the doc's compile variants so the consumer can route to the correct
+ // dataset-level path (a nil/empty Variants would fall back to the legacy
+ // unified path and tree nav could not be rebuilt). The variants are recovered
+ // from the doc-level compiled products' authoritative `compilation_template_kind_kwd`
+ // (via KindToVariant, O2a whitelist hard-fail), not re-derived from `compile_kwd`.
docs, err := c.docLister(ctx, tenant, kb)
if err != nil {
return fmt.Errorf("knowledge_compile: rebuild list docs: %w", err)
}
for _, docID := range docs {
- if perr := rs.Publish(ctx, tenant, kb, docID, string(EventTypeCompleted)); perr != nil {
+ variants, rerr := c.recoverDocVariants(ctx, tenant, kb, docID)
+ if rerr != nil {
+ // O2a hard failure: abort the rebuild so an unknown template kind
+ // cannot leave the doc's dataset-level products unrebuilt.
+ return fmt.Errorf("knowledge_compile: rebuild recover variants %s: %w", docID, rerr)
+ }
+ if perr := rs.Publish(ctx, tenant, kb, docID, string(EventTypeCompleted), variants); perr != nil {
return fmt.Errorf("knowledge_compile: rebuild republish %s: %w", docID, perr)
}
}
@@ -343,23 +362,77 @@ func defaultDocLister(ctx context.Context, tenant, kb string) ([]string, error)
return dao.NewDocumentDAO().ListIDsByKBIDWithOptions(ctx, kcDB, dao.DocumentListOptions{KbID: kb})
}
-// filterWikiPageCandidates returns only the candidates the dataset-level merge
-// is allowed to fold: for the wiki variant, that is strictly page-kind products
-// (Meta.kind == "page"). Sections are a doc-level concern and must never enter
-// the dataset-level page bucket, and the deduper must not carry an implicit
-// section-filter contract. Non-wiki variants pass through unchanged. Legacy wiki
-// rows whose kind is empty are derived in the Reader; only truly page-kind wiki
-// products proceed.
+// recoverDocVariants reconstructs the compile-type set a document produced, by
+// reading its persisted doc-level products and mapping each product's
+// authoritative `compilation_template_kind_kwd` through common.KindToVariant.
+// It is used by RebuildDataset (B1) so the republished completed events carry
+// the variants needed to route the dataset-level re-compile (tree nav, structure
+// merge, wiki merge) — a nil/empty Variants would fall back to the legacy
+// unified path and tree nav could not be rebuilt.
+//
+// KindToVariant (O2a) is a fixed whitelist that HARD-FAILS on unknown kinds: a
+// product whose kind is not in the whitelist is a corrupt/unsupported state and
+// the rebuild must abort (returning an error) rather than silently drop the
+// variant and leave the doc's dataset-level products unrebuilt. A product
+// without an authoritative kind falls back to its reverse-mapped variant (which
+// itself must be whitelist-valid, enforced by KwdToVariant in the Reader). The
+// returned slice is sorted and de-duplicated.
+func (c *Consumer) recoverDocVariants(ctx context.Context, tenant, kb, docID string) ([]string, error) {
+ products, err := c.reader.LoadDocProducts(ctx, tenant, kb, docID)
+ if err != nil {
+ return nil, fmt.Errorf("knowledge_compile: recover variants load doc %s: %w", docID, err)
+ }
+ seen := map[string]struct{}{}
+ var out []string
+ for _, p := range products {
+ // Authoritative: product.Kind (compilation_template_kind_kwd) when present.
+ if p.Kind != "" {
+ v, err := kccommon.KindToVariant(p.Kind)
+ if err != nil {
+ // O2a hard failure: do not continue with an incomplete variant set.
+ return nil, fmt.Errorf("knowledge_compile: recover variants doc %s kind %q: %w", docID, p.Kind, err)
+ }
+ if _, dup := seen[string(v)]; dup {
+ continue
+ }
+ seen[string(v)] = struct{}{}
+ out = append(out, string(v))
+ continue
+ }
+ // Fallback: the product's reverse-mapped variant (already whitelist-valid).
+ if p.Variant == "" {
+ // No authoritative kind and no reverse-mapped variant: this is a
+ // legacy/unknown row. Skip it so we never republish an empty variant
+ // (which the consumer cannot route and defeats "nil = legacy").
+ continue
+ }
+ if _, dup := seen[string(p.Variant)]; dup {
+ continue
+ }
+ seen[string(p.Variant)] = struct{}{}
+ out = append(out, string(p.Variant))
+ }
+ sort.Strings(out)
+ return out, nil
+}
+
+// filterWikiPageCandidates returns ONLY the wiki page products the dataset-level
+// wiki merge may fold. It drops every non-wiki product (tree/structure/nav) so
+// those never enter the wiki Dedup/KNN/WriteMerged path, and within the wiki
+// variant keeps strictly page-kind products (Meta.kind == "page"). Sections are
+// a doc-level concern and must never enter the dataset-level page bucket, and
+// the deduper must not carry an implicit section-filter contract. This is the
+// per-variant product selection (review issue 2): a completed wiki-only doc whose
+// event carried tree/structure products on a prior run must not feed stale
+// tree/structure rows into the wiki merge.
func filterWikiPageCandidates(candidates []kccommon.Product) []kccommon.Product {
- // Allocate a fresh slice: reusing candidates[:0] would overwrite the caller's
- // backing array in place, corrupting any other reference (e.g. the vector
- // audit that still reads `incoming`).
filtered := make([]kccommon.Product, 0, len(candidates))
for _, cand := range candidates {
- if cand.Variant == kccommon.VariantWiki {
- if metaString(cand.Meta, "kind") != "page" {
- continue
- }
+ if cand.Variant != kccommon.VariantWiki {
+ continue // tree/structure products never enter the wiki merge
+ }
+ if metaString(cand.Meta, "kind") != "page" {
+ continue
}
filtered = append(filtered, cand)
}
@@ -397,17 +470,19 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
// (deleted last) semantics we want.
type docState struct {
winEvent EventType
+ variants []string
}
byDoc := make(map[string]*docState, len(entries))
for _, e := range entries {
et := EventType(e.EventType)
st := byDoc[e.DocID]
if st == nil {
- byDoc[e.DocID] = &docState{winEvent: et}
+ byDoc[e.DocID] = &docState{winEvent: et, variants: e.Variants}
continue
}
// Later entries overwrite earlier ones: last event wins.
st.winEvent = et
+ st.variants = e.Variants
}
for docID, st := range byDoc {
switch st.winEvent {
@@ -428,7 +503,9 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
if _, hadTomb := tomb[docID]; hadTomb {
pendingTombClear = append(pendingTombClear, docID)
}
- completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted)})
+ // Preserve the winning event's variants so the dispatch below can
+ // route this doc to the matching dataset-level compile path (A0-4).
+ completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted), Variants: st.variants})
}
}
// Sort for deterministic iteration in the delete/load passes below.
@@ -518,6 +595,17 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
}
return err
}
+ // G3: structure ghost cleanup — remove structure dataset rows whose only
+ // source docs were just deleted. Also a locked destructive write.
+ if err := c.withWriteLock(ctx, kb, token, func() error {
+ return c.writer.DeleteStructureForDocs(ctx, tenant, kb, delIDs)
+ }); err != nil {
+ if errors.Is(err, errClaimSuperseded) {
+ common.Info("knowledge_compile: batch stale before structure ghost cleanup, aborting (rewrite barrier)",
+ zap.String("dataset_id", kb))
+ }
+ return err
+ }
}
// --- Completion merge ---
@@ -534,7 +622,44 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
if err != nil {
return err
}
- incoming = append(incoming, docProducts...)
+ // A0-4: the event's Variants are the authoritative dispatch signal — route
+ // ONLY the products whose variant the compiler actually produced for this
+ // doc. A doc re-compiled under a new template may still hold stale
+ // doc-level rows of an earlier variant in storage; without this gate those
+ // would leak into the nav/structure paths despite the event asking only
+ // for, say, wiki. Legacy events with empty Variants keep all products.
+ incoming = append(incoming, productsForVariants(docProducts, e.Variants)...)
+ }
+
+ // A0-4 dispatch: route tree/structure products to the dataset-navigation tree
+ // (NavService.UpsertDoc), and keep wiki (and only wiki) products on the
+ // existing dataset-level merge path below. This is the per-variant dispatch
+ // the plan requires: tree and structure both produce a nav by-product (B2),
+ // so both upsert their summary into the cross-document nav tree; wiki products
+ // continue to the replace-only merge.
+ navIn := navInputFromProducts(kb, incoming)
+ if len(navIn) > 0 {
+ ns := nav.GetNavService()
+ if ns == nil {
+ common.Warn("knowledge_compile: nav service unavailable, skipping dataset-nav upsert for batch",
+ zap.String("dataset_id", kb))
+ } else if err := c.upsertNavLocked(ctx, tenant, kb, token, ns, navIn); err != nil {
+ return err
+ }
+ }
+
+ // G1+G2: dataset-level structure merge and build-time stamp, both under the
+ // claim-fenced write lock (B7 applies to structure writes too): a worker
+ // invalidated by CancelInflight must not write structure rows after a rebuild
+ // has cleared them.
+ if err := c.withWriteLock(ctx, kb, token, func() error {
+ return c.mergeStructureDataset(ctx, tenant, kb, incoming)
+ }); err != nil {
+ if errors.Is(err, errClaimSuperseded) {
+ common.Info("knowledge_compile: batch stale before structure merge, aborting (rewrite barrier)",
+ zap.String("dataset_id", kb))
+ }
+ return err
}
// wiki_incremental port (M1): the dataset-level merge only processes wiki
@@ -785,3 +910,248 @@ func candidateIdentity(p kccommon.Product) string {
}
return "<unknown>"
}
+
+// productsForVariants filters a doc's loaded products down to those whose
+// variant is in the requested set. An empty set (legacy backlog entries without
+// variants, or a rebuild event that carries none) is treated as "all variants"
+// to preserve existing behavior; otherwise a product is kept only when its
+// variant appears in variants. This is what makes BacklogEntry.Variants actually
+// drive per-variant dispatch instead of handing every doc's full product set to
+// every dataset-level path.
+func productsForVariants(products []kccommon.Product, variants []string) []kccommon.Product {
+ if len(variants) == 0 {
+ return products
+ }
+ want := make(map[string]struct{}, len(variants))
+ for _, v := range variants {
+ want[v] = struct{}{}
+ }
+ out := products[:0:0]
+ for i := range products {
+ if _, ok := want[string(products[i].Variant)]; ok {
+ out = append(out, products[i])
+ }
+ }
+ return out
+}
+
+// navInputFromProducts extracts the dataset-navigation upsert inputs for the
+// tree and structure products in a batch (B2: nav trigger is tree || structure).
+// Summary extraction matches the component's by-product hooks:
+// - tree: the root product (Meta.kind=="root") carries the document summary in
+// Content and its vector in Vector.
+// - structure: the graph product (Meta.kind=="graph") carries the graph JSON in
+// Content; its entity descriptions are folded into a document-level summary
+// via pageIndexSummary.
+//
+// Each document contributes at most one nav input, keyed by DocID, so a doc that
+// yields several tree/structure products upserts once. NavService embeds the
+// summary itself when Embedd is empty, so the consumer needs no embedder.
+func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDocInput {
+ type acc struct {
+ in nav.UpsertDocInput
+ }
+ byDoc := make(map[string]*acc, len(products))
+ for i := range products {
+ p := &products[i]
+ switch p.Variant {
+ case kccommon.VariantTree:
+ if kind, _ := p.Meta["kind"].(string); kind != "root" {
+ continue
+ }
+ a := byDoc[p.DocID]
+ if a == nil {
+ byDoc[p.DocID] = &acc{in: nav.UpsertDocInput{TenantID: p.TenantID, KbID: kb, DocID: p.DocID}}
+ a = byDoc[p.DocID]
+ }
+ a.in.Summary = p.Content
+ a.in.Embedd = p.Vector
+ case kccommon.VariantStructure:
+ if kind, _ := p.Meta["kind"].(string); kind != "graph" {
+ continue
+ }
+ a := byDoc[p.DocID]
+ if a == nil {
+ byDoc[p.DocID] = &acc{in: nav.UpsertDocInput{TenantID: p.TenantID, KbID: kb, DocID: p.DocID}}
+ a = byDoc[p.DocID]
+ }
+ // Graph vector is NOT the summary vector; leave Embedd empty so
+ // NavService embeds the folded summary text.
+ a.in.Summary = pageIndexSummary(p.Content)
+ }
+ }
+ out := make([]nav.UpsertDocInput, 0, len(byDoc))
+ for _, a := range byDoc {
+ if strings.TrimSpace(a.in.Summary) == "" {
+ continue
+ }
+ out = append(out, a.in)
+ }
+ return out
+}
+
+// pageIndexSummary folds the entity descriptions of a structure graph JSON
+// ({"entities":[{"name","description"},...]}) into a document-level summary for
+// dataset navigation, matching the component's by-product logic.
+func pageIndexSummary(graphJSON string) string {
+ var graph struct {
+ Entities []struct {
+ Name string `json:"name"`
+ Description string `json:"description"`
+ } `json:"entities"`
+ }
+ if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {
+ return ""
+ }
+ var b strings.Builder
+ for _, e := range graph.Entities {
+ desc := strings.Join(strings.Fields(e.Description), " ")
+ if desc == "" {
+ continue
+ }
+ if e.Name != "" {
+ b.WriteString(e.Name)
+ b.WriteString(": ")
+ }
+ b.WriteString(desc)
+ b.WriteString("\n")
+ }
+ return b.String()
+}
+
+// mergeStructureDataset performs the dataset-level structure merge (G1/G4): it
+// groups the structure products of a batch by (name, type), folds their
+// descriptions and source doc/chunk sets into one StructureBucket per group, and
+// writes scope_kwd="dataset" rows via WriteMergedStructure. This mirrors Python
+// dataset_structure_merger._merge_bucket but is a self-contained from-scratch
+// path (B3: the legacy unified merge never aggregated structure by name/type).
+func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string, products []kccommon.Product) error {
+ // Bucket by (lower(name), type) for entities and (lower(from), lower(to))
+ // for relations — relations have no "name" and must not be silently dropped
+ // (review issue 3).
+ type ekey struct{ name, typ string }
+ type rkey struct{ from, to string }
+ entByKey := make(map[ekey]*StructureBucket, 16)
+ relByKey := make(map[rkey]*StructureBucket, 16)
+ for _, p := range products {
+ if p.Variant != kccommon.VariantStructure {
+ continue
+ }
+ kind := metaString(p.Meta, "kind")
+ from := metaString(p.Meta, "from")
+ to := metaString(p.Meta, "to")
+ if kind == "relation" || (from != "" && to != "") {
+ // Skip a relation with an incomplete endpoint pair: writing a bucket
+ // like "A -> " would let multiple malformed relations collapse into
+ // the same dataset row.
+ if from == "" || to == "" {
+ continue
+ }
+ k := rkey{from: strings.ToLower(from), to: strings.ToLower(to)}
+ b := relByKey[k]
+ if b == nil {
+ b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to}
+ relByKey[k] = b
+ }
+ appendBucket(b, p)
+ continue
+ }
+ // entity bucket
+ name := metaString(p.Meta, "name")
+ typ := metaString(p.Meta, "entity_type")
+ if typ == "" {
+ typ = metaString(p.Meta, "type")
+ }
+ if name == "" {
+ continue
+ }
+ k := ekey{name: strings.ToLower(name), typ: typ}
+ b := entByKey[k]
+ if b == nil {
+ b = &StructureBucket{Name: name, Type: typ}
+ entByKey[k] = b
+ }
+ appendBucket(b, p)
+ }
+ buckets := make([]StructureBucket, 0, len(entByKey)+len(relByKey))
+ for _, b := range entByKey {
+ buckets = append(buckets, *b)
+ }
+ for _, b := range relByKey {
+ buckets = append(buckets, *b)
+ }
+ if len(buckets) == 0 {
+ return nil
+ }
+ sort.Slice(buckets, func(i, j int) bool {
+ if buckets[i].Name != buckets[j].Name {
+ return buckets[i].Name < buckets[j].Name
+ }
+ return buckets[i].Type < buckets[j].Type
+ })
+ return c.writer.WriteMergedStructure(ctx, tenant, kb, buckets)
+}
+
+// appendBucket folds a structure product into a bucket: concatenates its
+// description, unions its source doc/chunk ids, and folds its vector into a true
+// element-wise running mean (review issue 13). VecCount tracks how many vectors
+// have been folded so the mean is order-independent.
+func appendBucket(b *StructureBucket, p kccommon.Product) {
+ if strings.TrimSpace(p.Content) != "" {
+ if b.Description != "" {
+ b.Description += "\n"
+ }
+ b.Description += strings.TrimSpace(p.Content)
+ }
+ b.SourceDocIDs = appendUnique(b.SourceDocIDs, []string{p.DocID})
+ b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, metaStringSlice(p.Meta, "source_chunk_ids"))
+ if len(p.Vector) > 0 {
+ b.VecCount++
+ if len(b.Vector) == 0 {
+ b.Vector = append([]float32(nil), p.Vector...)
+ } else {
+ n := len(p.Vector)
+ if len(b.Vector) < n {
+ b.Vector = append(b.Vector, make([]float32, n-len(b.Vector))...)
+ }
+ for i := 0; i < n; i++ {
+ // running mean: acc + (next - acc)/count
+ b.Vector[i] += (p.Vector[i] - b.Vector[i]) / float32(b.VecCount)
+ }
+ }
+ }
+}
+
+// appendUnique appends only values not already present, preserving order.
+func appendUnique(dst, values []string) []string {
+ seen := make(map[string]bool, len(dst)+len(values))
+ for _, v := range dst {
+ seen[v] = true
+ }
+ for _, v := range values {
+ if v == "" || seen[v] {
+ continue
+ }
+ seen[v] = true
+ dst = append(dst, v)
+ }
+ return dst
+}
+
+// upsertNavLocked places the batch's tree/structure summaries into the dataset
+// navigation tree under the per-dataset write/rebuild lock, with the claim-token
+// check inside the lock (B7: nav writes are claim-fenced like any other
+// destructive write so a stale worker cannot repopulate a rebuilt tree).
+func (c *Consumer) upsertNavLocked(ctx context.Context, tenant, kb, token string, ns nav.NavService, inputs []nav.UpsertDocInput) error {
+ return c.withWriteLock(ctx, kb, token, func() error {
+ for i := range inputs {
+ // The minimal-loop NavService.UpsertDoc is idempotent by summary; a
+ // failure here aborts the batch so the claim is not acked and the
+ // batch is retried.
+ if err := ns.UpsertDoc(ctx, inputs[i]); err != nil {
+ return fmt.Errorf("knowledge_compile: nav upsert %s: %w", inputs[i].DocID, err)
+ }
+ }
+ return nil
+ })
+}
diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.go
index 9c4090615a..fa949389af 100644
--- a/internal/ingestion/knowledge_compile/event.go
+++ b/internal/ingestion/knowledge_compile/event.go
@@ -103,21 +103,23 @@ func DefaultPublisher() Publisher { return defaultPublisher }
func DefaultClaimer() Claimer { return defaultClaimer }
// PublishCompleted records a doc_completed event: it appends the doc to the
-// KB's durable MySQL backlog and wakes idle workers over NATS. It is a no-op
-// when no Publisher has been installed (e.g. DB unavailable). A failure is
+// KB's durable MySQL backlog and wakes idle workers over NATS. variants carries
+// the doc-level compile types (tree/structure/wiki/mindmap) produced by this
+// doc so the consumer can dispatch to the matching dataset-level path. It is a
+// no-op when no Publisher has been installed (e.g. DB unavailable). A failure is
// returned so callers can log but never fail the pipeline on it.
-func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string) error {
+func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {
if defaultPublisher == nil {
return nil
}
- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted))
+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted), variants)
}
// PublishDeleted records a doc_deleted event the same way (Publish handles the
-// append + notify pairing).
+// append + notify pairing). Deleted events carry no compile types.
func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string) error {
if defaultPublisher == nil {
return nil
}
- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted))
+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), nil)
}
diff --git a/internal/ingestion/knowledge_compile/fake_engine_test.go b/internal/ingestion/knowledge_compile/fake_engine_test.go
index 48fe8e98ef..9e86549c09 100644
--- a/internal/ingestion/knowledge_compile/fake_engine_test.go
+++ b/internal/ingestion/knowledge_compile/fake_engine_test.go
@@ -33,6 +33,10 @@ type fakeEngine struct {
lastSearchReq *types.SearchRequest
// lastDeleteCond captures the most recent DeleteChunks condition.
lastDeleteCond map[string]interface{}
+ // deleteConds accumulates every DeleteChunks condition in call order, so a
+ // test can assert on a multi-delete clean (e.g. the full-set clean that
+ // sweeps compile_kwd buckets and the scope_kwd="dataset" bucket separately).
+ deleteConds []map[string]interface{}
// deleteCount is the number returned by DeleteChunks.
deleteCount int64
}
@@ -44,6 +48,7 @@ func (f *fakeEngine) Search(_ context.Context, req *types.SearchRequest) (*types
func (f *fakeEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, _, _ string) (int64, error) {
f.lastDeleteCond = condition
+ f.deleteConds = append(f.deleteConds, condition)
return f.deleteCount, nil
}
diff --git a/internal/ingestion/knowledge_compile/nav_dispatch_test.go b/internal/ingestion/knowledge_compile/nav_dispatch_test.go
new file mode 100644
index 0000000000..5fdd913fb4
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/nav_dispatch_test.go
@@ -0,0 +1,127 @@
+package knowledge_compile
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+ "ragflow/internal/service/nav"
+)
+
+// TestNavInputFromProducts_TreeAndStructure covers B2: nav is fed by BOTH tree
+// and structure products (not tree alone). Tree uses the root product's summary;
+// structure folds the graph entity descriptions into a summary via pageIndexSummary.
+func TestNavInputFromProducts_TreeAndStructure(t *testing.T) {
+ products := []kccommon.Product{
+ // tree: root product carries the doc summary + vector.
+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,
+ Content: "overall doc theme root summary", Vector: []float32{0.1, 0.2},
+ Meta: map[string]any{"kind": "root"}},
+ // tree: a non-root summary node is NOT a nav input.
+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree,
+ Content: "section body", Meta: map[string]any{"kind": "summary", "level": 0}},
+ // structure: graph product folds entity descriptions.
+ {DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,
+ Content: `{"entities":[{"name":"Engine","description":"a propulsion device"}]}`,
+ Meta: map[string]any{"kind": "graph", "compile_kwd": "page_index"}},
+ }
+
+ got := navInputFromProducts("kb1", products)
+ if len(got) != 2 {
+ t.Fatalf("want 2 nav inputs (tree d1 + structure d2), got %d", len(got))
+ }
+ byDoc := map[string]nav.UpsertDocInput{}
+ for _, in := range got {
+ byDoc[in.DocID] = in
+ }
+ treeIn, ok := byDoc["d1"]
+ if !ok {
+ t.Fatal("missing tree nav input for d1")
+ }
+ if !strings.Contains(treeIn.Summary, "overall doc theme") {
+ t.Errorf("tree summary = %q, want root summary", treeIn.Summary)
+ }
+ if !reflect.DeepEqual(treeIn.Embedd, []float32{0.1, 0.2}) {
+ t.Errorf("tree embedd should be the root vector, got %v", treeIn.Embedd)
+ }
+ structIn, ok := byDoc["d2"]
+ if !ok {
+ t.Fatal("missing structure nav input for d2")
+ }
+ if !strings.Contains(structIn.Summary, "Engine: a propulsion device") {
+ t.Errorf("structure summary = %q, want folded entity descriptions", structIn.Summary)
+ }
+ // Structure graph vector is NOT the summary vector: leave Embedd empty so
+ // NavService embeds the folded summary text.
+ if len(structIn.Embedd) != 0 {
+ t.Errorf("structure embedd should be empty (NavService re-embeds), got %v", structIn.Embedd)
+ }
+}
+
+// TestNavInputFromProducts_WikiExcluded covers the dispatch boundary: wiki
+// products never become nav inputs (they go to the replace-only wiki merge).
+func TestNavInputFromProducts_WikiExcluded(t *testing.T) {
+ products := []kccommon.Product{
+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantWiki,
+ Content: "a wiki page", Meta: map[string]any{"kind": "page", "slug": "entity/alpha"}},
+ }
+ got := navInputFromProducts("kb1", products)
+ if len(got) != 0 {
+ t.Fatalf("wiki products must not feed nav, got %d inputs", len(got))
+ }
+}
+
+// TestNavInputFromProducts_EmptySummaryDropped covers the guard: a doc whose
+// tree/structure summary is empty (no root / no graph) is skipped, not fed empty.
+func TestNavInputFromProducts_EmptySummaryDropped(t *testing.T) {
+ products := []kccommon.Product{
+ // tree root with empty content -> skip.
+ {DocID: "d1", TenantID: "t1", Variant: kccommon.VariantTree, Meta: map[string]any{"kind": "root"}},
+ // structure graph with no entity descriptions -> skip.
+ {DocID: "d2", TenantID: "t1", Variant: kccommon.VariantStructure,
+ Content: `{"entities":[]}`, Meta: map[string]any{"kind": "graph"}},
+ }
+ got := navInputFromProducts("kb1", products)
+ if len(got) != 0 {
+ t.Fatalf("empty-summary products must be dropped, got %d inputs", len(got))
+ }
+}
+
+// TestProductsForVariants_GatesDispatch covers the A0-4 per-variant gate: a doc
+// whose event carries only ["wiki"] must NOT have its stale tree/structure
+// doc-level products leak into the nav/structure paths. Empty variants (legacy
+// events) keep everything.
+func TestProductsForVariants_GatesDispatch(t *testing.T) {
+ products := []kccommon.Product{
+ {DocID: "d1", Variant: kccommon.VariantTree, Meta: map[string]any{"kind": "root"}, Content: "tree summary"},
+ {DocID: "d1", Variant: kccommon.VariantStructure, Meta: map[string]any{"kind": "graph"}, Content: `{"entities":[]}`},
+ {DocID: "d1", Variant: kccommon.VariantWiki, Meta: map[string]any{"kind": "page"}, Content: "wiki page"},
+ }
+ // A wiki-only event must drop the tree/structure products.
+ got := productsForVariants(products, []string{string(kccommon.VariantWiki)})
+ if len(got) != 1 || got[0].Variant != kccommon.VariantWiki {
+ t.Fatalf("wiki-only event: want exactly the wiki product, got %d", len(got))
+ }
+ // A tree-only event drops structure + wiki.
+ got = productsForVariants(products, []string{string(kccommon.VariantTree)})
+ if len(got) != 1 || got[0].Variant != kccommon.VariantTree {
+ t.Fatalf("tree-only event: want exactly the tree product, got %d", len(got))
+ }
+ // An empty variant set (legacy event / no variants) keeps everything.
+ if got = productsForVariants(products, nil); len(got) != 3 {
+ t.Fatalf("empty variants: want all 3 products kept, got %d", len(got))
+ }
+}
+
+// TestPageIndexSummary_ConcatenatesAndSkipsEmpty verifies pageIndexSummary folds
+// entity descriptions and skips empty ones.
+func TestPageIndexSummary_ConcatenatesAndSkipsEmpty(t *testing.T) {
+ got := pageIndexSummary(`{"entities":[{"name":"A","description":"one thing"},{"name":"B","description":""}]}`)
+ if !strings.Contains(got, "A: one thing") {
+ t.Errorf("summary = %q, want entity A", got)
+ }
+ if strings.Contains(got, "B") {
+ t.Errorf("summary should skip empty-description entity, got %q", got)
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index 809605b86f..6861ba35db 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -79,6 +79,12 @@ var compiledSelectFields = []string{
"name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
"slug_kwd", "type",
"kc_kind", "create_timestamp_flt", "create_time",
+ "compilation_template_kind_kwd",
+ // The product kind discriminator for structure/tree: the component stores it
+ // under knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd
+ // (tree: root/summary). Without these the reader cannot restore Meta["kind"]
+ // and the dataset-nav dispatch would skip structure/tree products (B2).
+ "knowledge_graph_kwd", "raptor_kwd",
}
// wikiSelectFields are the additional columns a wiki page carries (beyond
@@ -136,8 +142,8 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID str
continue
}
// Reverse-map the row's compile_kwd; reject dirty/unknown kinds
- // (kwdToVariant error) so a malformed row never loads as a product.
- rowVariant, verr := kwdToVariant(asString(c["compile_kwd"]))
+ // (KwdToVariant error) so a malformed row never loads as a product.
+ rowVariant, verr := KwdToVariant(asString(c["compile_kwd"]))
if verr != nil {
continue
}
@@ -176,7 +182,7 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID str
// content_with_weight) and the embedding from the q_<dim>_vec column.
//
// expect is the variant the caller is querying for. The stored compile_kwd is
-// reverse-mapped via kwdToVariant and compared against expect; a mismatch (or
+// reverse-mapped via KwdToVariant and compared against expect; a mismatch (or
// an unknown/dirty compile_kwd that does not map to any known variant) causes
// the row to be skipped. This is the canonical dirty-row contract promised by
// the plan: we never rely on the raw string equality alone, so unknown kinds
@@ -196,9 +202,9 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
// reverse-mapped consistently instead of being rejected as a dirty row.
variant := asString(c["compile_kwd"])
// Dirty-row contract: the raw compile_kwd must reverse-map to the expected
- // variant. An unknown/dirty kwd (kwdToVariant error) or a mapped variant
+ // variant. An unknown/dirty kwd (KwdToVariant error) or a mapped variant
// that differs from expect is rejected.
- mapped, err := kwdToVariant(variant)
+ mapped, err := KwdToVariant(variant)
if err != nil || mapped != expect {
return kccommon.Product{}, false
}
@@ -253,7 +259,17 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
if v, ok := c["type"].(string); ok && v != "" {
meta["type"] = v
}
- if _, ok := meta["kind"]; !ok {
+ // Restore the structure/tree product kind. The component stores it under
+ // knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd (tree:
+ // root/summary), and the dataset-nav dispatch (B2) keys off Meta["kind"] to
+ // pick the root/graph summary. Prefer these over the generic entity default.
+ // Use asString (not a bare type assertion): the engine may return a
+ // list-wrapped keyword column (review Major).
+ if v := asString(c["knowledge_graph_kwd"]); v != "" {
+ meta["kind"] = v
+ } else if v := asString(c["raptor_kwd"]); v != "" {
+ meta["kind"] = v
+ } else if _, ok := meta["kind"]; !ok {
if _, hasName := meta["name"]; hasName {
meta["kind"] = "entity"
}
@@ -265,7 +281,7 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
// wiki_section -> "section". This fix is what stops the processBatch
// "Meta.kind==page" filter from deleting every wiki page (previously kind
// was empty for wiki pages that had no entity/relation endpoint).
- if v, ok := c["kc_kind"].(string); ok && v != "" {
+ if v := asString(c["kc_kind"]); v != "" {
meta["kind"] = v
} else if variant == compileKwdWikiPage {
meta["kind"] = "page"
@@ -290,11 +306,16 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
}
vec, _ := kccommon.VectorFromChunkMap(c, 0)
+ // Restore the authoritative template kind (compilation_template_kind_kwd)
+ // so callers (e.g. RebuildDataset's variant recovery, B1a) can map it back
+ // via KindToVariant instead of re-deriving from the ambiguous compile_kwd.
+ kind := asString(c["compilation_template_kind_kwd"])
return kccommon.Product{
ID: id,
DocID: docID,
TenantID: tenant,
Variant: expect,
+ Kind: kind,
Content: content,
Vector: vec,
Meta: meta,
@@ -353,7 +374,7 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, vari
// The KNN Filter already scopes compile_kwd, but a foreign/legacy row that
// slipped past it must not poison the merge candidate. Reject by the
// reverse-mapped variant (the dirty-row contract): productFromChunkMap
- // validates kwdToVariant(c) == variant and drops dirty/unknown kinds. The
+ // validates KwdToVariant(c) == variant and drops dirty/unknown kinds. The
// raw-keyword check below is a fast pre-filter before the full product
// reconstruction.
if asString(c["compile_kwd"]) != expectKwd {
diff --git a/internal/ingestion/knowledge_compile/rebuild_variants_test.go b/internal/ingestion/knowledge_compile/rebuild_variants_test.go
new file mode 100644
index 0000000000..cef9c7b949
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/rebuild_variants_test.go
@@ -0,0 +1,71 @@
+package knowledge_compile
+
+import (
+ "context"
+ "reflect"
+ "testing"
+
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+)
+
+// fakeReader is a Reader double that returns a canned product set for
+// recoverDocVariants tests.
+type fakeReader struct {
+ products []kccommon.Product
+}
+
+func (f *fakeReader) LoadDocProducts(context.Context, string, string, string) ([]kccommon.Product, error) {
+ return f.products, nil
+}
+
+func (f *fakeReader) SearchSimilar(context.Context, string, string, kccommon.Variant, []float64, int, float64) (kccommon.Product, float64, error) {
+ return kccommon.Product{}, 0, nil
+}
+
+// TestRecoverDocVariants_AuthoritativeKind covers B1a/O2a: the authoritative
+// Product.Kind (compilation_template_kind_kwd) is mapped through KindToVariant;
+// results are sorted/deduped.
+func TestRecoverDocVariants_AuthoritativeKind(t *testing.T) {
+ c := &Consumer{reader: &fakeReader{products: []kccommon.Product{
+ {DocID: "d1", Kind: "structure"},
+ {DocID: "d1", Kind: "tree"},
+ // duplicate variant, deduped
+ {DocID: "d1", Kind: "structure"},
+ }}}
+ got, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1")
+ if err != nil {
+ t.Fatalf("recoverDocVariants error: %v", err)
+ }
+ want := []string{string(kccommon.VariantStructure), string(kccommon.VariantTree)}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("recoverDocVariants = %v, want %v", got, want)
+ }
+}
+
+// TestRecoverDocVariants_UnknownKindHardFails covers O2a: a whitelist-out
+// authoritative kind aborts recovery (returns an error) so the rebuild does not
+// proceed with an incomplete variant set.
+func TestRecoverDocVariants_UnknownKindHardFails(t *testing.T) {
+ c := &Consumer{reader: &fakeReader{products: []kccommon.Product{
+ {DocID: "d1", Kind: "structure"},
+ {DocID: "d1", Kind: "garbage"},
+ }}}
+ if _, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1"); err == nil {
+ t.Fatal("recoverDocVariants must hard-fail on an unknown authoritative kind (O2a)")
+ }
+}
+
+// TestRecoverDocVariants_FallbackVariant covers B1a: a product without an
+// authoritative kind falls back to its reverse-mapped variant.
+func TestRecoverDocVariants_FallbackVariant(t *testing.T) {
+ c := &Consumer{reader: &fakeReader{products: []kccommon.Product{
+ {DocID: "d1", Variant: kccommon.VariantWiki},
+ }}}
+ got, err := c.recoverDocVariants(context.Background(), "t1", "kb1", "d1")
+ if err != nil {
+ t.Fatalf("recoverDocVariants error: %v", err)
+ }
+ if len(got) != 1 || got[0] != string(kccommon.VariantWiki) {
+ t.Fatalf("recoverDocVariants fallback = %v, want [wiki]", got)
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.go
index bf9d7f136b..c2c4f14ad1 100644
--- a/internal/ingestion/knowledge_compile/scheduler.go
+++ b/internal/ingestion/knowledge_compile/scheduler.go
@@ -57,9 +57,17 @@ const (
// can re-apply the same tombstone handling as the broker-based design without
// re-reading the queue. The backlog is an ordered log; the last event for a
// given doc (by append order) wins, so no sequence number is needed.
+//
+// Variants records the compile types (tree/structure/wiki/mindmap) produced by
+// the doc-level compile for this doc, so the consumer can group a claimed batch
+// into per-variant sub-batches and dispatch each to its own dataset-level
+// compile path. It is empty (nil) for deleted events and for events published
+// before this field existed; the consumer treats a nil/empty Variants as
+// "unknown/legacy" and falls back to the legacy unified path.
type BacklogEntry struct {
- DocID string `json:"doc_id"`
- EventType string `json:"event_type"`
+ DocID string `json:"doc_id"`
+ EventType string `json:"event_type"`
+ Variants []string `json:"variants,omitempty"`
}
// ClaimResult is returned by Scheduler.Claim: the KB's tenant plus the closed
@@ -83,7 +91,9 @@ type Publisher interface {
// Publish records one doc event in the dataset's durable backlog and wakes
// idle workers. It is transactional so concurrent publishers do not clobber
// each other's backlog, and it always pairs the append with a notify.
- Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error
+ // variants carries the doc-level compile types (tree/structure/wiki/mindmap)
+ // for completed events; it is nil for deleted events.
+ Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error
}
// Claimer is the consumer-side role (Option E §11.5). A cluster of competing
@@ -203,8 +213,8 @@ func (s *mysqlScheduler) Provision(ctx context.Context) error {
// workers. The append is transactional (concurrent publishers do not clobber
// each other's backlog), and the notify is always paired with the append so a
// producer never needs a separate Notify call.
-func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error {
- return s.publish(ctx, tenantID, datasetID, docID, eventType)
+func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {
+ return s.publish(ctx, tenantID, datasetID, docID, eventType, variants)
}
// loadRow fetches the dataset scheduling row (no lock). Returns nil (no error)
@@ -220,11 +230,11 @@ func (s *mysqlScheduler) loadRow(ctx context.Context, datasetID string) (*entity
return &row, nil
}
-func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error {
+func (s *mysqlScheduler) publish(ctx context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {
if s.db == nil {
return nil
}
- entry := BacklogEntry{DocID: docID, EventType: eventType}
+ entry := BacklogEntry{DocID: docID, EventType: eventType, Variants: variants}
// KnowledgeCompileDataset.dataset_id is the PRIMARY KEY, so the SELECT ... FOR
// UPDATE below also takes an InnoDB gap lock for a not-yet-existing dataset;
// concurrent publishers for the same dataset serialize on that lock and the
@@ -642,17 +652,23 @@ func (s *mysqlScheduler) SubscribeNotify(ctx context.Context) (<-chan string, er
// doc_id + event_type). The batch is the exact set the worker claimed, so
// this is a precise removal, not a "clear all" — any inflight added by a
// concurrent claim is preserved.
+//
+// The key is doc_id+"\x00"+event_type: BacklogEntry now carries a Variants
+// slice which is not comparable, so it cannot be used as a Go map key. Variants
+// deliberately does not participate in the removal match — Ack still removes by
+// doc_id + event_type so an entry whose Variants differ from the inflight copy
+// is still removed exactly once.
func removeEntries(inflight, batch []BacklogEntry) []BacklogEntry {
if len(batch) == 0 {
return inflight
}
- drop := make(map[BacklogEntry]bool, len(batch))
+ drop := make(map[string]bool, len(batch))
for _, e := range batch {
- drop[e] = true
+ drop[e.DocID+"\x00"+e.EventType] = true
}
out := make([]BacklogEntry, 0, len(inflight))
for _, e := range inflight {
- if drop[e] {
+ if drop[e.DocID+"\x00"+e.EventType] {
continue
}
out = append(out, e)
@@ -699,11 +715,11 @@ func NewFakeScheduler() *FakeScheduler {
func (f *FakeScheduler) Provision(_ context.Context) error { return nil }
// Publish appends one doc event and pushes a notify (same contract as MySQL).
-func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string) error {
- return f.publish(tenantID, datasetID, docID, eventType)
+func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string, variants []string) error {
+ return f.publish(tenantID, datasetID, docID, eventType, variants)
}
-func (f *FakeScheduler) publish(tenantID, datasetID, docID, eventType string) error {
+func (f *FakeScheduler) publish(tenantID, datasetID, docID, eventType string, variants []string) error {
f.mu.Lock()
defer f.mu.Unlock()
r, ok := f.rows[datasetID]
@@ -714,7 +730,7 @@ func (f *FakeScheduler) publish(tenantID, datasetID, docID, eventType string) er
if r.tenant == "" {
r.tenant = tenantID
}
- r.backlog = append(r.backlog, BacklogEntry{DocID: docID, EventType: eventType})
+ r.backlog = append(r.backlog, BacklogEntry{DocID: docID, EventType: eventType, Variants: variants})
// Mirror the MySQL scheduler: surface pending unless a worker is running.
if r.state != DatasetStateRunning {
r.state = DatasetStatePending
diff --git a/internal/ingestion/knowledge_compile/scheduler_test.go b/internal/ingestion/knowledge_compile/scheduler_test.go
new file mode 100644
index 0000000000..dc413dee21
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/scheduler_test.go
@@ -0,0 +1,106 @@
+package knowledge_compile
+
+import (
+ "encoding/json"
+ "reflect"
+ "testing"
+)
+
+// TestRemoveEntriesMatchesByDocAndEventType locks the A0-1 contract: BacklogEntry
+// now carries a Variants slice (not comparable), so removeEntries must key on
+// DocID+EventType only and must not be affected by Variants. Two entries with the
+// same doc_id+event_type but different variants must collapse to a single removal,
+// and an entry whose Variants differ from the inflight copy must still be removed
+// exactly once.
+func TestRemoveEntriesMatchesByDocAndEventType(t *testing.T) {
+ inflight := []BacklogEntry{
+ {DocID: "d1", EventType: "doc_completed", Variants: []string{"tree", "wiki"}},
+ {DocID: "d2", EventType: "doc_completed", Variants: []string{"structure"}},
+ {DocID: "d3", EventType: "doc_deleted"},
+ }
+
+ // Batch claims d1 (with a variants subset different from inflight) and d3.
+ batch := []BacklogEntry{
+ {DocID: "d1", EventType: "doc_completed", Variants: []string{"wiki"}},
+ {DocID: "d3", EventType: "doc_deleted"},
+ }
+
+ out := removeEntries(inflight, batch)
+
+ if len(out) != 1 || out[0].DocID != "d2" {
+ t.Fatalf("removeEntries should keep only d2, got %+v", out)
+ }
+}
+
+// TestRemoveEntriesEmptyBatch verifies removeEntries with an empty batch is a no-op.
+func TestRemoveEntriesEmptyBatch(t *testing.T) {
+ inflight := []BacklogEntry{{DocID: "d1", EventType: "doc_completed", Variants: []string{"tree"}}}
+ out := removeEntries(inflight, nil)
+ if !reflect.DeepEqual(out, inflight) {
+ t.Fatalf("empty batch should preserve inflight, got %+v", out)
+ }
+}
+
+// TestBacklogEntryJSONBackwardCompat locks the A0-1 JSON contract: an old backlog
+// row serialized without the "variants" field must unmarshal with Variants == nil
+// (empty, not an error), which the consumer treats as "legacy/unknown" and falls
+// back to the unified path.
+func TestBacklogEntryJSONBackwardCompat(t *testing.T) {
+ old := []byte(`{"doc_id":"d1","event_type":"doc_completed"}`)
+ var e BacklogEntry
+ if err := json.Unmarshal(old, &e); err != nil {
+ t.Fatalf("unmarshal legacy backlog: %v", err)
+ }
+ if e.DocID != "d1" || e.EventType != "doc_completed" {
+ t.Fatalf("legacy fields not restored: %+v", e)
+ }
+ if e.Variants != nil {
+ t.Fatalf("legacy backlog should unmarshal Variants as nil, got %v", e.Variants)
+ }
+
+ // A completed event with variants round-trips intact.
+ withVariants := []byte(`{"doc_id":"d1","event_type":"doc_completed","variants":["tree","wiki"]}`)
+ if err := json.Unmarshal(withVariants, &e); err != nil {
+ t.Fatalf("unmarshal variants backlog: %v", err)
+ }
+ if !reflect.DeepEqual(e.Variants, []string{"tree", "wiki"}) {
+ t.Fatalf("variants not restored: %+v", e.Variants)
+ }
+ // omitempty: a nil Variants serializes without the key (matches legacy format).
+ enc, err := json.Marshal(BacklogEntry{DocID: "d2", EventType: "doc_deleted"})
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ var decoded map[string]any
+ if err := json.Unmarshal(enc, &decoded); err != nil {
+ t.Fatalf("re-marshal decode: %v", err)
+ }
+ if _, hasVariants := decoded["variants"]; hasVariants {
+ t.Fatalf("nil Variants should be omitted, got %s", enc)
+ }
+}
+
+// TestFakeSchedulerPublishCarriesVariants locks the A0-2 contract: FakeScheduler
+// records the variants on the backlog entry, so the consumer sees them.
+func TestFakeSchedulerPublishCarriesVariants(t *testing.T) {
+ f := NewFakeScheduler()
+ if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), []string{"tree", "structure"}); err != nil {
+ t.Fatalf("publish: %v", err)
+ }
+ if err := f.Publish(t.Context(), "t1", "kb1", "d2", string(EventTypeDeleted), nil); err != nil {
+ t.Fatalf("publish deleted: %v", err)
+ }
+ res, ok, err := f.Claim(t.Context(), "kb1")
+ if err != nil || !ok {
+ t.Fatalf("claim: ok=%v err=%v", ok, err)
+ }
+ if len(res.Entries) != 2 {
+ t.Fatalf("want 2 backlog entries, got %d", len(res.Entries))
+ }
+ if !reflect.DeepEqual(res.Entries[0].Variants, []string{"tree", "structure"}) {
+ t.Fatalf("completed variants not carried: %+v", res.Entries[0])
+ }
+ if res.Entries[1].Variants != nil {
+ t.Fatalf("deleted event should carry nil variants, got %+v", res.Entries[1])
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.go
new file mode 100644
index 0000000000..764a9b8b9c
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go
@@ -0,0 +1,204 @@
+package knowledge_compile
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+)
+
+// fakeWriter is a Writer double that records the last WriteMergedStructure
+// buckets so a test can assert the grouping behavior without an engine.
+type fakeWriter struct {
+ buckets []StructureBucket
+}
+
+func (f *fakeWriter) WriteMerged(context.Context, string, string, []kccommon.Product) error {
+ return nil
+}
+func (f *fakeWriter) WriteMergedStructure(_ context.Context, _, _ string, buckets []StructureBucket) error {
+ f.buckets = buckets
+ return nil
+}
+func (f *fakeWriter) DeleteMergedForVariant(context.Context, string, string, []kccommon.Variant) error {
+ return nil
+}
+func (f *fakeWriter) DeleteStructureForDocs(context.Context, string, string, []string) error {
+ return nil
+}
+func (f *fakeWriter) DeleteMerged(context.Context, string, string) error { return nil }
+func (f *fakeWriter) DeleteDocLevelForDocs(context.Context, string, string, []string) error {
+ return nil
+}
+func (f *fakeWriter) StripMergedSources(context.Context, string, string, []string) error {
+ return nil
+}
+func (f *fakeWriter) ProjectWikiGraph(context.Context, string, string) error { return nil }
+func (f *fakeWriter) DropWikiGraph(context.Context, string, string) error { return nil }
+
+// TestMergeStructureDataset_GroupsByNameType covers G1: structure products are
+// bucketed by (name, type), descriptions folded, source ids unioned, and only
+// scope_kwd="dataset" structure rows written.
+func TestMergeStructureDataset_GroupsByNameType(t *testing.T) {
+ c := &Consumer{writer: &fakeWriter{}}
+ products := []kccommon.Product{
+ {Variant: kccommon.VariantStructure, DocID: "d1",
+ Content: "Engine desc A",
+ Meta: map[string]any{"name": "Engine", "entity_type": "component", "source_chunk_ids": []string{"c1"}}},
+ {Variant: kccommon.VariantStructure, DocID: "d2",
+ Content: "Engine desc B",
+ Meta: map[string]any{"name": "Engine", "entity_type": "component", "source_chunk_ids": []string{"c2"}}},
+ // different type -> separate bucket
+ {Variant: kccommon.VariantStructure, DocID: "d3",
+ Content: "Fuel desc",
+ Meta: map[string]any{"name": "Fuel", "entity_type": "substance", "source_chunk_ids": []string{"c3"}}},
+ // non-structure product ignored
+ {Variant: kccommon.VariantWiki, DocID: "d4", Content: "wiki page",
+ Meta: map[string]any{"name": "Ignored", "kind": "page"}},
+ }
+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {
+ t.Fatalf("mergeStructureDataset: %v", err)
+ }
+ fw := c.writer.(*fakeWriter)
+ if len(fw.buckets) != 2 {
+ t.Fatalf("want 2 buckets (Engine + Fuel), got %d", len(fw.buckets))
+ }
+ var engineBucket, fuelBucket *StructureBucket
+ for i := range fw.buckets {
+ switch fw.buckets[i].Name {
+ case "Engine":
+ engineBucket = &fw.buckets[i]
+ case "Fuel":
+ fuelBucket = &fw.buckets[i]
+ }
+ }
+ if engineBucket == nil || fuelBucket == nil {
+ t.Fatalf("missing expected buckets: %+v", fw.buckets)
+ }
+ if engineBucket.Type != "component" {
+ t.Errorf("Engine type = %q, want component", engineBucket.Type)
+ }
+ if !strings.Contains(engineBucket.Description, "Engine desc A") || !strings.Contains(engineBucket.Description, "Engine desc B") {
+ t.Errorf("Engine descriptions not folded: %q", engineBucket.Description)
+ }
+ if len(engineBucket.SourceDocIDs) != 2 {
+ t.Errorf("Engine source doc union = %v, want [d1 d2]", engineBucket.SourceDocIDs)
+ }
+}
+
+// TestMergeStructureDataset_RelationsNotDropped covers review issue 3: structure
+// relation products (kind=relation with from/to, no name) must be folded into a
+// dataset-level relation bucket instead of being silently dropped.
+func TestMergeStructureDataset_RelationsNotDropped(t *testing.T) {
+ c := &Consumer{writer: &fakeWriter{}}
+ products := []kccommon.Product{
+ {Variant: kccommon.VariantStructure, DocID: "d1", Content: "rel desc",
+ Meta: map[string]any{"kind": "relation", "from": "A", "to": "B", "source_chunk_ids": []string{"c1"}}},
+ {Variant: kccommon.VariantStructure, DocID: "d2", Content: "rel desc 2",
+ Meta: map[string]any{"kind": "relation", "from": "A", "to": "B", "source_chunk_ids": []string{"c2"}}},
+ }
+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {
+ t.Fatalf("mergeStructureDataset: %v", err)
+ }
+ fw := c.writer.(*fakeWriter)
+ if len(fw.buckets) != 1 {
+ t.Fatalf("want 1 relation bucket, got %d", len(fw.buckets))
+ }
+ b := fw.buckets[0]
+ if b.Type != "relation" || b.FromEntity != "A" || b.ToEntity != "B" {
+ t.Errorf("relation bucket = %+v, want Type=relation From=A To=B", b)
+ }
+ if len(b.SourceDocIDs) != 2 {
+ t.Errorf("relation source doc union = %v, want [d1 d2]", b.SourceDocIDs)
+ }
+}
+
+// TestDatasetLevelStructureID_StableAndCaseInsensitive covers G1: the id is
+// deterministic and case-insensitive on name so merges hit the same row.
+func TestDatasetLevelStructureID_StableAndCaseInsensitive(t *testing.T) {
+ a := datasetLevelStructureID("t1", "kb1", "Engine", "component")
+ b := datasetLevelStructureID("t1", "kb1", "engine", "component")
+ if a != b {
+ t.Errorf("structure id should be case-insensitive on name: %q vs %q", a, b)
+ }
+ if a == datasetLevelStructureID("t1", "kb1", "Fuel", "component") {
+ t.Errorf("different names must yield different ids")
+ }
+}
+
+// TestKwdToVariant_MapsStructureSubKinds covers B1a: structure doc-level products
+// stamp their inferred compile kind verbatim (hypergraph/list/set/timeline/
+// page_index/...), and KwdToVariant must reverse-map each to VariantStructure so
+// the reader does NOT drop them before the dataset-level merge / nav dispatch.
+func TestKwdToVariant_MapsStructureSubKinds(t *testing.T) {
+ for _, kwd := range []string{
+ "hypergraph", "list", "set", "timeline", "page_index",
+ "session_essence", "session_graph", "knowledge_graph", "graph", "structure",
+ } {
+ v, err := KwdToVariant(kwd)
+ if err != nil {
+ t.Errorf("KwdToVariant(%q) unexpected error: %v", kwd, err)
+ continue
+ }
+ if v != kccommon.VariantStructure {
+ t.Errorf("KwdToVariant(%q) = %q, want structure", kwd, v)
+ }
+ }
+}
+
+// TestKwdToVariant_RejectsUnknown still hard-fails on a non-whitelisted kwd
+// (O2a): a malformed or unknown compile_kwd is rejected, not silently folded.
+func TestKwdToVariant_RejectsUnknown(t *testing.T) {
+ for _, kwd := range []string{"garbage", "artifact_page", ""} {
+ if _, err := KwdToVariant(kwd); err == nil {
+ t.Errorf("KwdToVariant(%q) should error", kwd)
+ }
+ }
+}
+
+// TestProductFromChunkMap_RestoresStructureTreeKind covers the second-round fix:
+// the reader must restore Meta["kind"] for structure (from knowledge_graph_kwd)
+// and tree (from raptor_kwd) products so the dataset-nav dispatch (B2) can pick
+// the graph/root summary instead of skipping them.
+func TestProductFromChunkMap_RestoresStructureTreeKind(t *testing.T) {
+ // structure graph product: compile_kwd is the inferred sub-kind "list",
+ // kind is stored in knowledge_graph_kwd.
+ structRow := map[string]interface{}{
+ "id": "s1",
+ "doc_id": "d1",
+ "compile_kwd": "list",
+ "kc_payload": `{"entities":[]}`,
+ "knowledge_graph_kwd": "graph",
+ "name_kwd": "engine",
+ "source_chunk_ids": []interface{}{"c1"},
+ "source_doc_ids": []interface{}{"d1"},
+ "compilation_template_kind_kwd": "list",
+ }
+ sp, ok := productFromChunkMap(structRow, "t1", kccommon.VariantStructure)
+ if !ok {
+ t.Fatal("structure row should reconstruct")
+ }
+ if kind, _ := sp.Meta["kind"].(string); kind != "graph" {
+ t.Errorf("structure Meta.kind = %q, want graph (so nav dispatch can pick it)", kind)
+ }
+
+ // tree root product: kind stored in raptor_kwd.
+ treeRow := map[string]interface{}{
+ "id": "t1",
+ "doc_id": "d1",
+ "compile_kwd": "tree",
+ "kc_payload": "overall summary",
+ "raptor_kwd": "root",
+ "source_chunk_ids": []interface{}{"c1"},
+ "source_doc_ids": []interface{}{"d1"},
+ "compilation_template_kind_kwd": "tree",
+ }
+ tp, ok := productFromChunkMap(treeRow, "t1", kccommon.VariantTree)
+ if !ok {
+ t.Fatal("tree row should reconstruct")
+ }
+ if kind, _ := tp.Meta["kind"].(string); kind != "root" {
+ t.Errorf("tree Meta.kind = %q, want root (so nav dispatch can pick it)", kind)
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 97d2450540..88cc9514ef 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -60,6 +60,23 @@ type Writer interface {
ProjectWikiGraph(ctx context.Context, tenant, kb string) error
// DropWikiGraph deletes every wiki_entity / wiki_relation row for the dataset.
DropWikiGraph(ctx context.Context, tenant, kb string) error
+ // WriteMergedStructure writes the dataset-level structure merged rows
+ // (scope_kwd="dataset") for a KB, one row per (name, type) bucket, carrying
+ // the folded descriptions and the union of source docs/chunks (G1/G4).
+ WriteMergedStructure(ctx context.Context, tenant, kb string, buckets []StructureBucket) error
+ // DeleteStructureForDocs removes dataset-level structure rows (scope_kwd=
+ // "dataset", compile_kwd="structure") that reference a deleted doc and no
+ // longer have any remaining source doc (G3 ghost cleanup). Rows that still
+ // have other source docs are kept; their source_doc_ids are NOT stripped here
+ // (StripMergedSources handles the union-preserving edit).
+ DeleteStructureForDocs(ctx context.Context, tenant, kb string, deletedDocIDs []string) error
+ // DeleteMergedForVariant removes the dataset-level merged rows for a KB whose
+ // compile type is in variants (B4): structure (scope_kwd="dataset"),
+ // wiki (compile_kwd wiki_page/wiki_section), and nav (compile_kwd
+ // dataset_nav) as applicable. A full rebuild clears every variant the
+ // consumer manages (B1b: fixed full-set, empty set also clears all) so
+ // removed-template ghosts cannot survive.
+ DeleteMergedForVariant(ctx context.Context, tenant, kb string, variants []kccommon.Variant) error
// DeleteMerged removes the dataset-level merged rows for a KB so an
// incremental build can start from a clean slate. The structural filter
// deletes only rows produced by the dataset-level merge — kb_id == kb AND
@@ -70,6 +87,22 @@ type Writer interface {
DeleteMerged(ctx context.Context, tenant, kb string) error
}
+// StructureBucket is one dataset-level structure merge unit (G1/G4): a group of
+// a structure entity (Name/Type) OR a relation (FromEntity/ToEntity) with its
+// folded descriptions and the union of source docs/chunks. It is written as a
+// scope_kwd="dataset" row. Relations carry FromEntity/ToEntity instead of Name.
+type StructureBucket struct {
+ Name string
+ Type string
+ Description string // folded entity descriptions
+ SourceDocIDs []string // union of source doc ids
+ SourceChunkIDs []string // union of source chunk ids
+ Vector []float32
+ VecCount int // number of vectors folded into Vector (for true mean)
+ FromEntity string // relation only
+ ToEntity string // relation only
+}
+
// engineWriter persists dataset-level merged products through the global
// DocEngine (§11.7). Like engineReader, it depends on the process-wide DocEngine
// obtained via engine.Get(); the storage schema lives behind the engine
@@ -167,6 +200,218 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
return runCompilerJobs(ctx, jobs)
}
+// WriteMergedStructure writes the dataset-level structure merged rows for a KB
+// (G1/G4). Each StructureBucket is a scope_kwd="dataset" row with a stable
+// dataset-level id keyed on (name, type), the folded description, the union of
+// source docs/chunks, and the bucket vector. Rows are available_int=1 so the
+// dataset-level structure index is searchable, and compile_kwd="structure".
+func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb string, buckets []StructureBucket) error {
+ if len(buckets) == 0 {
+ return nil
+ }
+ eng := w.eng
+ if eng == nil {
+ eng = engine.Get()
+ }
+ if eng == nil {
+ return nil
+ }
+ baseName := fmt.Sprintf("ragflow_%s", tenant)
+ now := time.Now()
+ // Read-modify-write: an incremental batch must not drop the source docs/chunks
+ // an earlier batch already accumulated for a (name,type) bucket, so load the
+ // existing dataset rows by their stable id and union their sources (review
+ // issue 3 / #3 Major).
+ existing := map[string]StructureBucket{}
+ {
+ ids := make([]string, 0, len(buckets))
+ for _, b := range buckets {
+ if b.Name == "" {
+ continue
+ }
+ ids = append(ids, datasetLevelStructureID(tenant, kb, b.Name, b.Type))
+ }
+ if len(ids) > 0 {
+ res, err := eng.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{baseName},
+ KbIDs: []string{kb},
+ SelectFields: []string{"id", "source_doc_ids", "source_chunk_ids"},
+ Filter: map[string]interface{}{"kb_id": kb, "id": ids},
+ Limit: len(ids),
+ })
+ if err != nil {
+ return fmt.Errorf("structure merge read-modify-write load: %w", err)
+ }
+ for _, c := range res.Chunks {
+ id, _ := c["id"].(string)
+ if id == "" {
+ continue
+ }
+ existing[id] = StructureBucket{
+ SourceDocIDs: firstStringSlice(c["source_doc_ids"]),
+ SourceChunkIDs: firstStringSlice(c["source_chunk_ids"]),
+ }
+ }
+ }
+ }
+ rows := make([]map[string]interface{}, 0, len(buckets))
+ for _, b := range buckets {
+ desc := strings.TrimSpace(b.Description)
+ if desc == "" {
+ continue
+ }
+ if b.Name == "" {
+ continue
+ }
+ bid := datasetLevelStructureID(tenant, kb, b.Name, b.Type)
+ // Union the current batch's sources with any already-accumulated ones.
+ if prev, ok := existing[bid]; ok {
+ b.SourceDocIDs = appendUnique(b.SourceDocIDs, prev.SourceDocIDs)
+ b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, prev.SourceChunkIDs)
+ }
+ row := map[string]interface{}{
+ // Stable dataset-level id keyed on the (name, type) or (from, to) bucket.
+ "id": bid,
+ "doc_id": kb,
+ "tenant_id": tenant,
+ "kb_id": kb,
+ "available_int": 1,
+ "compile_kwd": compileKwdStructure,
+ "scope_kwd": "dataset",
+ "content_with_weight": desc,
+ "kc_payload": desc,
+ "source_doc_ids": b.SourceDocIDs,
+ "source_chunk_ids": b.SourceChunkIDs,
+ "create_time": now.Format("2006-01-02 15:04:05"),
+ "create_timestamp_flt": float64(now.Unix()),
+ }
+ if b.FromEntity != "" || b.ToEntity != "" {
+ // relation row: carries from/to entities; kind=relation, no name_kwd.
+ row["type_kwd"] = "relation"
+ row["from_entity_kwd"] = b.FromEntity
+ row["to_entity_kwd"] = b.ToEntity
+ } else {
+ row["type_kwd"] = "entity"
+ row["name_kwd"] = b.Name
+ row["entity_type_kwd"] = b.Type
+ }
+ if len(b.Vector) > 0 {
+ row["q_"+fmt.Sprintf("%d", len(b.Vector))+"_vec"] = f32ToF64Slice(b.Vector)
+ }
+ rows = append(rows, row)
+ }
+ if len(rows) == 0 {
+ return nil
+ }
+ // Insert in bounded batches, matching writeMergedBatchSize.
+ jobs := make([]CompilerJob, 0, (len(rows)+writeMergedBatchSize-1)/writeMergedBatchSize)
+ for start := 0; start < len(rows); start += writeMergedBatchSize {
+ end := start + writeMergedBatchSize
+ if end > len(rows) {
+ end = len(rows)
+ }
+ batch := rows[start:end]
+ jobs = append(jobs, func() error {
+ _, err := eng.InsertChunks(ctx, batch, baseName, kb)
+ return err
+ })
+ }
+ return runCompilerJobs(ctx, jobs)
+}
+
+// DeleteStructureForDocs removes dataset-level structure rows whose source docs
+// are all gone (G3, mirroring Python dataset_structure_merger._cleanup_deleted_docs).
+// A structure dataset row that still has any non-deleted source doc survives; a
+// row whose source_doc_ids are a subset of the deleted set is a ghost and is
+// removed. This is best-effort in the sense that rows without any source_doc_ids
+// are untouched (they may predate source tracking).
+func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb string, deletedDocIDs []string) error {
+ if len(deletedDocIDs) == 0 {
+ return nil
+ }
+ eng := w.eng
+ if eng == nil {
+ eng = engine.Get()
+ }
+ if eng == nil {
+ return nil
+ }
+ baseName := fmt.Sprintf("ragflow_%s", tenant)
+ deleted := make(map[string]bool, len(deletedDocIDs))
+ for _, d := range deletedDocIDs {
+ deleted[d] = true
+ }
+ // Page through structure dataset rows (mirroring StripMergedSources) so a
+ // dataset with more than one page of rows does not leave ghosts past the cap
+ // surviving silently (review Minor).
+ const pageSize = 500
+ var ghostIDs []string
+ for offset := 0; ; offset += pageSize {
+ res, err := eng.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{baseName},
+ KbIDs: []string{kb},
+ SelectFields: []string{"id", "source_doc_ids"},
+ Filter: map[string]interface{}{"kb_id": kb, "scope_kwd": "dataset", "compile_kwd": compileKwdStructure},
+ Offset: offset,
+ Limit: pageSize,
+ })
+ if err != nil {
+ return fmt.Errorf("structure ghost scan: %w", err)
+ }
+ if len(res.Chunks) == 0 {
+ break
+ }
+ for _, c := range res.Chunks {
+ id, _ := c["id"].(string)
+ if id == "" {
+ continue
+ }
+ srcs := firstStringSlice(c["source_doc_ids"])
+ if len(srcs) == 0 {
+ continue // no source tracking; not safe to declare a ghost
+ }
+ allGone := true
+ for _, s := range srcs {
+ if !deleted[s] {
+ allGone = false
+ break
+ }
+ }
+ if allGone {
+ ghostIDs = append(ghostIDs, id)
+ }
+ }
+ if len(res.Chunks) < pageSize {
+ break
+ }
+ }
+ if len(ghostIDs) == 0 {
+ return nil
+ }
+ _, err := eng.DeleteChunks(ctx, map[string]interface{}{"id": ghostIDs, "kb_id": kb}, baseName, kb)
+ if err != nil {
+ return fmt.Errorf("structure ghost cleanup: %w", err)
+ }
+ return nil
+}
+
+// datasetLevelStructureID builds the stable dataset-level id for a structure
+// bucket, keyed on (name, type). It must be deterministic so an incremental
+// merge read-modify-writes the same row (and a rebuild clean removes it).
+func datasetLevelStructureID(tenant, kb, name, typ string) string {
+ return "dataset_structure_" + hashStr(tenant+"\x00"+kb+"\x00"+strings.ToLower(name)+"\x00"+typ)
+}
+
+// f32ToF64Slice converts a float32 vector to float64 for the engine's dense
+// vector column (the engine stores q_*_vec as float64).
+func f32ToF64Slice(v []float32) []float64 {
+ out := make([]float64, len(v))
+ for i, x := range v {
+ out[i] = float64(x)
+ }
+ return out
+}
+
// mergedChunkMap builds the chunk-index document for a dataset-level merged
// product. It uses the dataset-level idempotency key (§11.6) as `id`, never the
// per-doc key, and is always available_int=1 (searchable).
@@ -181,11 +426,16 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
srcDocIDs := metaStringSlice(p.Meta, "source_doc_ids")
srcChunkIDs := metaStringSlice(p.Meta, "source_chunk_ids")
m := map[string]interface{}{
- "id": datasetLevelID(tenant, kb, p),
- "doc_id": kb,
- "tenant_id": tenant,
- "kb_id": kb,
- "available_int": 1,
+ "id": datasetLevelID(tenant, kb, p),
+ "doc_id": kb,
+ "tenant_id": tenant,
+ "kb_id": kb,
+ "available_int": 1,
+ // scope_kwd marks this row as dataset-level (O1=B). It is the unified
+ // doc/dataset discriminator across all compile types; wiki merged rows now
+ // carry scope_kwd="dataset" alongside available_int=1 so the consumer and
+ // clean paths can filter by scope instead of (only) available_int.
+ "scope_kwd": "dataset",
"compile_kwd": compileKwdForVariant(p.Variant),
"content_with_weight": p.Content,
"kc_payload": p.Content, // raw payload, for Reader reconstruction
@@ -446,15 +696,17 @@ func metaString(m map[string]any, key string) string {
return ""
}
-func metaStringSlice(m map[string]any, key string) []string {
- switch v := m[key].(type) {
+// firstStringSlice extracts a []string from an engine row value, tolerating both
+// []string and []any forms; it returns nil when the value is not a string slice.
+func firstStringSlice(v any) []string {
+ switch s := v.(type) {
case []string:
- return v
+ return s
case []any:
- out := make([]string, 0, len(v))
- for _, e := range v {
- if s, ok := e.(string); ok {
- out = append(out, s)
+ out := make([]string, 0, len(s))
+ for _, x := range s {
+ if str, ok := x.(string); ok {
+ out = append(out, str)
}
}
return out
@@ -462,6 +714,10 @@ func metaStringSlice(m map[string]any, key string) []string {
return nil
}
+func metaStringSlice(m map[string]any, key string) []string {
+ return firstStringSlice(m[key])
+}
+
// metaInt extracts an integer from a map value that may be boxed as float64
// (JSON number), int64, string, or a typed int — the engine/JSON round-trip does
// not guarantee a single numeric type.
@@ -503,19 +759,32 @@ func metaFloat(m map[string]any, key string) (float64, bool) {
return 0, false
}
-// kwdToVariant is the inverse of compileKwdForVariant: it maps a stored
+// KwdToVariant is the inverse of compileKwdForVariant: it maps a stored
// compile_kwd back to its compiler Variant. Both wiki_page and wiki_section
// map to VariantWiki (same product family); the page/section distinction is
// carried by the kc_kind field, not the variant. Returns an error for an
-// unknown kwd so callers can reject dirty/foreign rows.
-func kwdToVariant(kwd string) (kccommon.Variant, error) {
+// unknown kwd so callers can reject dirty/foreign rows. Structure products
+// stamp the inferred compile kind verbatim (list/set/hypergraph), which are NOT
+// in the KindToVariant whitelist, so they are mapped to VariantStructure
+// explicitly before the whitelist lookup; unknown kinds hard-fail (O2a).
+func KwdToVariant(kwd string) (kccommon.Variant, error) {
switch kwd {
case compileKwdWikiPage, compileKwdWikiSection, compileKwdWikiEntity, compileKwdWikiRelation:
return kccommon.VariantWiki, nil
- case string(kccommon.VariantStructure), string(kccommon.VariantTree), string(kccommon.VariantMindmap):
+ case string(kccommon.VariantTree), string(kccommon.VariantMindmap):
return kccommon.Variant(kwd), nil
}
- return "", fmt.Errorf("unknown compile_kwd %q", kwd)
+ // Structure products stamp the inferred compile kind verbatim (hypergraph /
+ // list / set / timeline / page_index / graph / ... — see structure.InferType),
+ // NOT the collapsed "structure" variant. Map the three fixed structure compile
+ // kinds plus any whitelisted template kind through KindToVariant (O2a) so the
+ // reader reconstructs structure products instead of dropping them as "unknown
+ // kwd" (B1a). Unknown kinds hard-fail.
+ switch kwd {
+ case "list", "set", "hypergraph":
+ return kccommon.VariantStructure, nil
+ }
+ return kccommon.KindToVariant(kwd)
}
// --- Wiki page graph materialization (wiki_entity / wiki_relation) ---
@@ -531,6 +800,11 @@ const (
compileKwdWikiEntity = "wiki_entity"
compileKwdWikiRelation = "wiki_relation"
compileKwdWikiPageGraph = "wiki_page_graph" // legacy Python blob, swept on drop
+ // compileKwdStructure tags structure dataset-level merged rows
+ // (scope_kwd="dataset"); compileKwdNav tags the dataset-navigation rows
+ // written by NavService. Both are targets of per-variant clean (B4).
+ compileKwdStructure = "structure"
+ compileKwdNav = "dataset_nav"
)
// wikiGraphBatchSize bounds how many graph rows each parallel InsertChunks call
@@ -640,6 +914,17 @@ func (w engineWriter) DropWikiGraph(ctx context.Context, tenant, kb string) erro
// the source of truth; it never targets per-document rows (available_int=0) nor
// rows of other tenants / variants, so a wrong tenantID / kb cannot cascade.
func (w engineWriter) DeleteMerged(ctx context.Context, tenant, kb string) error {
+ return w.DeleteMergedForVariant(ctx, tenant, kb, []kccommon.Variant{kccommon.VariantWiki})
+}
+
+// DeleteMergedForVariant deletes the dataset-level merged rows for a KB across
+// the given compile variants (B4). When variants is empty it clears the full set
+// the consumer manages (B1b: a full rebuild always clears everything, so an
+// empty set still means "clear all"). Row scope: structure dataset rows are
+// tagged scope_kwd="dataset"; wiki merged rows carry available_int=1; nav rows
+// carry compile_kwd="dataset_nav". The filter union is OR-ed across variants so
+// one call clears every relevant row in a single engine delete.
+func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb string, variants []kccommon.Variant) error {
eng := w.eng
if eng == nil {
eng = engine.Get()
@@ -648,16 +933,76 @@ func (w engineWriter) DeleteMerged(ctx context.Context, tenant, kb string) error
return nil
}
baseName := fmt.Sprintf("ragflow_%s", tenant)
- _, err := eng.DeleteChunks(ctx, map[string]interface{}{
- "kb_id": kb,
- "available_int": 1,
- "compile_kwd": []string{
- compileKwdWikiPage,
- compileKwdWikiSection,
- },
- }, baseName, kb)
- if err != nil {
- return fmt.Errorf("delete merged: %w", err)
+ if len(variants) == 0 {
+ // Full-set clean (B1b): everything the consumer manages. Structure
+ // dataset rows are tagged scope_kwd="dataset" (not a fixed compile_kwd),
+ // so the full clean deletes by kb_id + compile_kwd IN (the fixed-kwd
+ // buckets) OR scope_kwd="dataset".
+ _, err := eng.DeleteChunks(ctx, map[string]interface{}{
+ "kb_id": kb,
+ "compile_kwd": []string{
+ compileKwdNav,
+ compileKwdWikiPage,
+ compileKwdWikiSection,
+ compileKwdStructure,
+ },
+ }, baseName, kb)
+ if err != nil {
+ return fmt.Errorf("delete merged (all variants): %w", err)
+ }
+ // structure dataset rows carry scope_kwd="dataset" + compile_kwd="structure";
+ // sweep them too (idempotent with the compile_kwd filter above).
+ _, err = eng.DeleteChunks(ctx, map[string]interface{}{
+ "kb_id": kb,
+ "scope_kwd": "dataset",
+ }, baseName, kb)
+ if err != nil {
+ return fmt.Errorf("delete merged (structure scope): %w", err)
+ }
+ return nil
+ }
+ // Issue ONE delete per distinct variant bucket rather than one AND-ed filter:
+ // different variants target different columns (wiki: available_int=1 +
+ // compile_kwd; structure: scope_kwd="dataset"; nav: compile_kwd="dataset_nav"
+ // with available_int=0), and AND-ing them would exclude the others' rows.
+ // RebuildDataset (B1b) passes the full managed set, so this per-bucket sweep
+ // is correct for both full and per-variant rebuilds.
+ for _, v := range variants {
+ switch v {
+ case kccommon.VariantWiki:
+ // wiki merged rows are tagged available_int=1 (distinct from
+ // doc-level available_int=0); scope the wiki delete to them only.
+ if _, err := eng.DeleteChunks(ctx, map[string]interface{}{
+ "kb_id": kb,
+ "available_int": 1,
+ "compile_kwd": []string{compileKwdWikiPage, compileKwdWikiSection},
+ }, baseName, kb); err != nil {
+ return fmt.Errorf("delete merged (wiki): %w", err)
+ }
+ case kccommon.VariantStructure:
+ // structure dataset rows are scope_kwd="dataset" + compile_kwd=
+ // "structure". The compile_kwd is required: wiki merged rows ALSO carry
+ // scope_kwd="dataset" (W5), so a scope-only sweep would wrongly delete
+ // wiki merged rows too (review Major).
+ if _, err := eng.DeleteChunks(ctx, map[string]interface{}{
+ "kb_id": kb,
+ "scope_kwd": "dataset",
+ "compile_kwd": []string{compileKwdStructure},
+ }, baseName, kb); err != nil {
+ return fmt.Errorf("delete merged (structure): %w", err)
+ }
+ case kccommon.VariantTree:
+ // tree/nav rows are compile_kwd="dataset_nav" with available_int=0;
+ // do NOT add available_int=1 (that would exclude them).
+ if _, err := eng.DeleteChunks(ctx, map[string]interface{}{
+ "kb_id": kb,
+ "compile_kwd": []string{compileKwdNav},
+ }, baseName, kb); err != nil {
+ return fmt.Errorf("delete merged (nav): %w", err)
+ }
+ case kccommon.VariantMindmap:
+ // mindmap has no dataset-level merged rows in the consumer path
+ }
}
return nil
}
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index 9d445b2f89..9056c1970b 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -142,9 +142,165 @@ func TestDeleteMergedScopesToMergedWikiRows(t *testing.T) {
}
}
+// TestDeleteMergedForVariant_EmptySetClearsAll covers B4/B1b: an empty variants
+// slice (full rebuild) clears the full managed set — nav + wiki + structure
+// compile_kwds — plus the structure scope_kwd="dataset" sweep.
+func TestDeleteMergedForVariant_EmptySetClearsAll(t *testing.T) {
+ eng := &fakeEngine{}
+ w := engineWriter{eng: eng}
+ if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1", nil); err != nil {
+ t.Fatalf("DeleteMergedForVariant: %v", err)
+ }
+ // The full clean issues two deletes: the compile_kwd bucket first, then the
+ // scope_kwd="dataset" sweep. Assert on the first (compile_kwd) delete.
+ if len(eng.deleteConds) < 1 {
+ t.Fatal("engine.DeleteChunks was not called")
+ }
+ kwds, ok := eng.deleteConds[0]["compile_kwd"].([]string)
+ if !ok {
+ t.Fatalf("compile_kwd filter not a string slice, got %T", eng.deleteConds[0]["compile_kwd"])
+ }
+ want := []string{compileKwdNav, compileKwdWikiPage, compileKwdWikiSection, compileKwdStructure}
+ if len(kwds) != len(want) {
+ t.Fatalf("full clean compile_kwd = %v, want %v", kwds, want)
+ }
+ got := map[string]bool{}
+ for _, k := range kwds {
+ got[k] = true
+ }
+ for _, k := range want {
+ if !got[k] {
+ t.Errorf("full clean missing compile_kwd %q", k)
+ }
+ }
+ // The second delete sweeps structure scope_kwd="dataset".
+ foundScope := false
+ for _, cond := range eng.deleteConds {
+ if cond["scope_kwd"] == "dataset" {
+ foundScope = true
+ }
+ }
+ if !foundScope {
+ t.Error("full clean must also sweep scope_kwd=dataset")
+ }
+}
+
+// TestDeleteMergedForVariant_WikiScopesToAvailableOne covers the per-variant
+// wiki clean: it must issue a delete scoped to available_int=1 and the wiki
+// page/section kwds (wiki merged rows carry available_int=1).
+func TestDeleteMergedForVariant_WikiScopesToAvailableOne(t *testing.T) {
+ eng := &fakeEngine{}
+ w := engineWriter{eng: eng}
+ if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1", []kccommon.Variant{kccommon.VariantWiki}); err != nil {
+ t.Fatalf("DeleteMergedForVariant: %v", err)
+ }
+ if len(eng.deleteConds) != 1 {
+ t.Fatalf("wiki clean should issue exactly one delete, got %d", len(eng.deleteConds))
+ }
+ cond := eng.deleteConds[0]
+ if cond["available_int"] != 1 {
+ t.Errorf("wiki clean must scope to available_int=1, got %v", cond["available_int"])
+ }
+ kwds, ok := cond["compile_kwd"].([]string)
+ if !ok || len(kwds) != 2 {
+ t.Fatalf("wiki clean compile_kwd = %v, want [wiki_page wiki_section]", cond["compile_kwd"])
+ }
+}
+
+// TestDeleteMergedForVariant_StructureScopesToDataset covers the per-variant
+// structure clean: dataset rows are tagged scope_kwd="dataset" (compile_kwd may
+// be the inferred sub-kind), so the delete is scoped to scope_kwd="dataset".
+func TestDeleteMergedForVariant_StructureScopesToDataset(t *testing.T) {
+ eng := &fakeEngine{}
+ w := engineWriter{eng: eng}
+ if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1", []kccommon.Variant{kccommon.VariantStructure}); err != nil {
+ t.Fatalf("DeleteMergedForVariant: %v", err)
+ }
+ if len(eng.deleteConds) != 1 {
+ t.Fatalf("structure clean should issue exactly one delete, got %d", len(eng.deleteConds))
+ }
+ if eng.deleteConds[0]["scope_kwd"] != "dataset" {
+ t.Errorf("structure clean must scope to scope_kwd=dataset, got %v", eng.deleteConds[0]["scope_kwd"])
+ }
+}
+
+// TestDeleteMergedForVariant_WikiPlusNavCoversNavRows covers the combined-clean
+// bug: each variant issues its OWN delete (one AND-ed filter would exclude the
+// other's rows — nav rows carry available_int=0, structure rows scope_kwd=
+// "dataset"). A wiki+tree clean must produce one wiki delete (available_int=1)
+// and one nav delete (compile_kwd=dataset_nav, no available_int).
+func TestDeleteMergedForVariant_WikiPlusNavCoversNavRows(t *testing.T) {
+ eng := &fakeEngine{}
+ w := engineWriter{eng: eng}
+ if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1",
+ []kccommon.Variant{kccommon.VariantWiki, kccommon.VariantTree}); err != nil {
+ t.Fatalf("DeleteMergedForVariant: %v", err)
+ }
+ if len(eng.deleteConds) != 2 {
+ t.Fatalf("wiki+tree clean should issue 2 deletes (wiki + nav), got %d", len(eng.deleteConds))
+ }
+ var wikiCond, navCond map[string]interface{}
+ for _, cond := range eng.deleteConds {
+ kwds, _ := cond["compile_kwd"].([]string)
+ for _, k := range kwds {
+ if k == compileKwdNav {
+ navCond = cond
+ }
+ }
+ if cond["available_int"] == 1 {
+ wikiCond = cond
+ }
+ }
+ if wikiCond == nil {
+ t.Error("expected a wiki delete scoped to available_int=1")
+ }
+ if navCond == nil {
+ t.Error("expected a nav delete (compile_kwd=dataset_nav)")
+ } else if _, hasAvail := navCond["available_int"]; hasAvail {
+ t.Errorf("nav delete must not carry available_int (nav rows are 0), got %v", navCond["available_int"])
+ }
+}
+
+// TestDeleteStructureForDocs_RemovesGhostOnly covers G3: structure dataset rows
+// whose source docs are ALL deleted are ghosts and removed; rows that still have
+// a surviving source doc are kept.
+func TestDeleteStructureForDocs_RemovesGhostOnly(t *testing.T) {
+ eng := &fakeEngine{searchChunks: []map[string]interface{}{
+ {"id": "row_ghost", "source_doc_ids": []any{"d1", "d2"}}, // all deleted -> ghost
+ {"id": "row_keep", "source_doc_ids": []any{"d1", "d3"}}, // d3 survives -> keep
+ }}
+ w := engineWriter{eng: eng}
+ if err := w.DeleteStructureForDocs(context.Background(), "t1", "kb1", []string{"d1", "d2"}); err != nil {
+ t.Fatalf("DeleteStructureForDocs: %v", err)
+ }
+ if eng.lastDeleteCond == nil {
+ t.Fatal("DeleteChunks not called")
+ }
+ ids, ok := eng.lastDeleteCond["id"].([]string)
+ if !ok || len(ids) != 1 || ids[0] != "row_ghost" {
+ t.Fatalf("ghost ids = %v, want [row_ghost]", eng.lastDeleteCond["id"])
+ }
+}
+
+// TestDeleteStructureForDocs_NoGhostsNoDelete covers G3: when every structure row
+// still has a surviving source doc, no delete is issued.
+func TestDeleteStructureForDocs_NoGhostsNoDelete(t *testing.T) {
+ eng := &fakeEngine{searchChunks: []map[string]interface{}{
+ {"id": "row_keep", "source_doc_ids": []any{"d1", "d3"}},
+ }}
+ before := len(eng.deleteConds)
+ w := engineWriter{eng: eng}
+ if err := w.DeleteStructureForDocs(context.Background(), "t1", "kb1", []string{"d1"}); err != nil {
+ t.Fatalf("DeleteStructureForDocs: %v", err)
+ }
+ if len(eng.deleteConds) != before {
+ t.Fatalf("no ghost should mean no delete, got %d deletes", len(eng.deleteConds)-before)
+ }
+}
+
// TestProductFromChunkMapRejectsDirtyKwd locks the dirty-row contract from the
// wiki_incremental plan (Claim 4): productFromChunkMap must reverse-map the raw
-// compile_kwd via kwdToVariant and reject any row whose kwd does not map to the
+// compile_kwd via KwdToVariant and reject any row whose kwd does not map to the
// expected variant, including unknown / malformed kinds (e.g. "artifact_page",
// "garbage", empty). It must not silently fall back to a raw-string comparison.
func TestProductFromChunkMapRejectsDirtyKwd(t *testing.T) {
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 6dafb7a03a..7139ad1a8b 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -21,6 +21,7 @@ import (
"encoding/json"
"fmt"
"ragflow/internal/utility"
+ "sort"
"strings"
"time"
@@ -29,6 +30,7 @@ import (
"ragflow/internal/engine"
"ragflow/internal/entity"
"ragflow/internal/ingestion/component"
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/ingestion/knowledge_compile"
pipelinepkg "ragflow/internal/ingestion/pipeline"
indexdoc "ragflow/internal/ingestion/task/indexdoc"
@@ -263,7 +265,13 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
// (available_int=1). The notification is sent only after a successful persist
// and is best-effort / non-fatal — a delivery failure is logged but does not
// fail the pipeline task.
- if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID); err != nil {
+ //
+ // The variants passed to PublishCompleted are the compile types this document
+ // produced, derived from the authoritative `compilation_template_kind_kwd` the
+ // KnowledgeCompiler component stamps on each compiled product (the resolved
+ // template's kind → KindToVariant, O2a whitelist). This is the compiler's
+ // runtime inference surfaced here, NOT a re-derivation from `compile_kwd`.
+ if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, compiledVariants(chunks)); err != nil {
common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))
}
@@ -314,6 +322,74 @@ func markCompiledProductsHidden(chunks []map[string]any) {
}
}
+// compiledVariants returns the sorted, de-duplicated set of compile types a
+// document's compiled products carry. It reads the authoritative
+// `compilation_template_kind_kwd` the KnowledgeCompiler component stamps on each
+// compiled product (the resolved template's kind) and maps it through
+// common.KindToVariant (O2a whitelist: unknown kinds are skipped). Products
+// without an authoritative kind fall back to their `compile_kwd`-derived variant.
+// This surfaces the compiler's runtime variant inference to PublishCompleted so
+// the consumer can route the dataset-level re-compile per compile type.
+func compiledVariants(chunks []map[string]any) []string {
+ seen := map[string]struct{}{}
+ for _, ck := range chunks {
+ if _, ok := ck["compile_kwd"]; !ok {
+ continue
+ }
+ var v kccommon.Variant
+ if kind, ok := ck["compilation_template_kind_kwd"].(string); ok && kind != "" {
+ mapped, err := kccommon.KindToVariant(kind)
+ if err != nil {
+ continue // unknown template kind (O2a): skip, do not mis-route
+ }
+ v = mapped
+ } else {
+ // Fallback on compile_kwd via the shared knowledge_compile mapping
+ // (which folds the inferred structure kinds list/set/hypergraph into
+ // VariantStructure before the KindToVariant whitelist lookup).
+ mapped, err := knowledge_compile.KwdToVariant(asCompiledKwd(ck))
+ if err != nil {
+ continue
+ }
+ v = mapped
+ }
+ if _, dup := seen[string(v)]; dup {
+ continue
+ }
+ seen[string(v)] = struct{}{}
+ }
+ if len(seen) == 0 {
+ return nil
+ }
+ out := make([]string, 0, len(seen))
+ for k := range seen {
+ out = append(out, k)
+ }
+ sort.Strings(out)
+ return out
+}
+
+// asCompiledKwd extracts the compile_kwd keyword value from a chunk map,
+// tolerating both a bare string and a list-wrapped keyword column from the
+// engine. It returns "" when absent.
+func asCompiledKwd(c map[string]any) string {
+ switch v := c["compile_kwd"].(type) {
+ case string:
+ return v
+ case []string:
+ if len(v) > 0 {
+ return v[0]
+ }
+ case []any:
+ if len(v) > 0 {
+ if s, ok := v[0].(string); ok {
+ return s
+ }
+ }
+ }
+ return ""
+}
+
func (s *PipelineExecutor) recordPipelineLog(ctx context.Context, db *gorm.DB, docID, dsl, status string) {
var dslMap entity.JSONMap
if err := json.Unmarshal([]byte(dsl), &dslMap); err != nil {
diff --git a/internal/service/nav/nav.go b/internal/service/nav/nav.go
index 9b590ac2c3..a87fcfd8c9 100644
--- a/internal/service/nav/nav.go
+++ b/internal/service/nav/nav.go
@@ -55,6 +55,20 @@ type UpsertDocInput struct {
Embedd []float32 // optional precomputed embedding
}
+// NavMergeLLM summarizes or merges nav text via an LLM. It mirrors the Python
+// dataset_nav._llm_merge/_llm_create_summary calls. Keeping it a small interface
+// (rather than depending on the agent chat layer) lets the nlp package wire the
+// production chat invoker without an import cycle; nil disables LLM behavior and
+// the implementation falls back to deterministic naming.
+type NavMergeLLM interface {
+ // Merge returns a merged cluster description for one or more source texts
+ // (temperature 0.1, mirroring Python _llm_merge).
+ Merge(ctx context.Context, tenantID string, texts []string) (string, error)
+ // CreateSummary returns a short cluster name + summary for a source text
+ // (mirroring Python _llm_create_summary). Return name="" to fall back.
+ CreateSummary(ctx context.Context, tenantID string, text string) (name, summary string, err error)
+}
+
// NavService is the single read/write entrypoint for a dataset's navigation
// tree. It is the only nav consumer used by agent tools, REST handlers and the
// tree/structure compile-complete hooks.
diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.go
index 2f8c133f47..cf057f8795 100644
--- a/internal/service/nlp/datasetnav.go
+++ b/internal/service/nlp/datasetnav.go
@@ -39,6 +39,12 @@ const (
navRecurse = 0.65 // descend while sim >= this
navMinSim = 0.50 // sim >= this -> new sibling cluster
navMaxDepth = 6 // max descent levels during best-cluster search
+
+ // Split thresholds (mirror Python dataset_nav._MAX_FANOUT / _MAX_DOCS_PER_CLUSTER).
+ // Shared by maybeSplitCluster and the append-path guard so an append only pays
+ // the children-search cost when the cluster is plausibly overfull.
+ navMaxFanout = 64 // max direct children before rebalance
+ navMaxDocsPerCluster = 50 // max docs per leaf cluster before split
)
// NavEmbedder embeds text. tenantID lets a production implementation resolve
@@ -51,6 +57,7 @@ type NavEmbedder interface {
type NavService struct {
embed NavEmbedder
engine engine.DocEngine // optional; falls back to engine.Get() when nil
+ llm nav.NavMergeLLM // optional; nil disables LLM merge/summary
}
// NewNavService builds the ES-backed NavService. embed may be nil when the
@@ -59,6 +66,14 @@ func NewNavService(embed NavEmbedder) *NavService {
return &NavService{embed: embed}
}
+// SetNavMergeLLM installs the optional LLM used for cluster-description merging
+// and summary naming (mirroring Python dataset_nav._llm_merge/_llm_create_summary).
+// When nil (the default), NavService falls back to deterministic naming and
+// concatenated descriptions.
+func (s *NavService) SetNavMergeLLM(llm nav.NavMergeLLM) {
+ s.llm = llm
+}
+
func (s *NavService) docEngine() (engine.DocEngine, error) {
if s.engine != nil {
return s.engine, nil
@@ -290,8 +305,19 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
}
}
}
- // Changed summary: remove the old nav_doc first (no cascade in minimal loop).
- if _, err := s.deleteNavDoc(ctx, in.TenantID, in.KbID, in.DocID); err != nil {
+ // Changed summary: remove the old nav_doc first (and prune any emptied
+ // cluster). The doc may have lived in a cluster; the cluster is then
+ // decremented/cascaded by RemoveDoc's cleanup logic.
+ parent, err := s.deleteNavDoc(ctx, in.TenantID, in.KbID, in.DocID)
+ if err != nil {
+ return err
+ }
+ if parent != "" && parent != navRootParent {
+ if err := s.removeDocFromCluster(ctx, in.TenantID, in.KbID, parent, in.DocID); err != nil {
+ return err
+ }
+ }
+ if err := s.cleanupEmptyCluster(ctx, in.TenantID, in.KbID, parent); err != nil {
return err
}
}
@@ -304,10 +330,16 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
if bestName != "" && sim >= navMergeThreshold {
parent := bestName
- if err := s.appendDocToCluster(ctx, de, in.TenantID, in.KbID, bestName, in.DocID); err != nil {
+ if err := s.appendDocToCluster(ctx, de, in.TenantID, in.KbID, bestName, in.DocID,
+ fmt.Sprintf("q_%d_vec", len(vec))); err != nil {
return err
}
+ // Explicit stable id (A5): the nav_doc row is addressable by a
+ // deterministic id (hash of the "dataset_nav:doc:{doc_id}" key) so
+ // RemoveDoc / cascade cleanup can locate and delete it precisely,
+ // instead of relying on a doc_id + type filter alone.
_, err = de.InsertChunks(ctx, []map[string]interface{}{{
+ "id": navDocID(in.TenantID, in.KbID, in.DocID),
"compile_kwd": navCompileKwd,
"available_int": 0,
"type_kwd": "nav_doc",
@@ -335,8 +367,15 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
// than it, so depth = parentDepth+1 — not a hard-coded 1.
depth = bestDepth + 1
}
- name := navDocName(in.DocID, in.Summary)
+ // New cluster: when an LLM is configured, generate a short name + summary
+ // (Python _llm_create_summary); otherwise fall back to a deterministic name
+ // (A1, graceful without LLM).
+ name, summary := s.llmCreateSummary(ctx, in.TenantID, in.Summary)
+ if summary == "" {
+ summary = in.Summary
+ }
_, err = de.InsertChunks(ctx, []map[string]interface{}{{
+ "id": navClusterID(in.TenantID, in.KbID, name),
"compile_kwd": navCompileKwd,
"available_int": 0,
"type_kwd": "nav_cluster",
@@ -345,12 +384,77 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"depth_int": depth,
"doc_count_int": 1,
"doc_ids_kwd": []string{in.DocID},
- "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_cluster", "description": in.Summary}),
+ "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_cluster", "description": summary}),
"q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),
}}, idx, in.KbID)
return err
}
+// navDocID returns the explicit stable id for a nav_doc row (A5), a hash of the
+// canonical "dataset_nav:doc:{doc_id}" key. It is deterministic so deletes and
+// cascade cleanup can address the row without a doc_id+type filter scan.
+func navDocID(tenantID, kbID, docID string) string {
+ return "dataset_nav_doc_" + contentHash8(tenantID+"\x00"+kbID+"\x00"+docID)
+}
+
+// navClusterID returns the explicit stable id for a nav_cluster row (A5), a hash
+// of the canonical "dataset_nav:cluster:{kb}:{name}" key (mirroring Python
+// _nav_cluster_id).
+func navClusterID(tenantID, kbID, name string) string {
+ return "dataset_nav_cluster_" + contentHash8(tenantID+"\x00"+kbID+"\x00"+name)
+}
+
+// cleanupEmptyCluster deletes a nav_cluster that holds no docs and no child
+// clusters, then recurses to its parent (cascade, mirroring Python
+// _cleanup_empty_cluster). It is called by RemoveDoc after deleting a nav_doc so
+// an emptied cluster does not linger.
+func (s *NavService) cleanupEmptyCluster(ctx context.Context, tenantID, kbID, clusterName string) error {
+ if clusterName == "" || clusterName == navRootParent {
+ return nil
+ }
+ de, err := s.docEngine()
+ if err != nil {
+ return err
+ }
+ chunks, _, err := s.navSearch(ctx, tenantID, kbID,
+ navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),
+ []string{"id", "parent_kwd", "doc_count_int", "doc_ids_kwd"}, 0, 1, nil)
+ if err != nil {
+ return err
+ }
+ if len(chunks) == 0 {
+ return nil
+ }
+ // Children: count any nav_cluster/nav_doc whose parent is this cluster.
+ children, _, err := s.navSearch(ctx, tenantID, kbID,
+ navFilter(map[string]interface{}{"parent_kwd": []string{clusterName}}),
+ []string{"id"}, 0, 1, nil)
+ if err != nil {
+ return err
+ }
+ if len(children) > 0 {
+ return nil // not empty: has children
+ }
+ parent := firstStringValue(chunks[0]["parent_kwd"])
+ if intValue(chunks[0]["doc_count_int"]) > 0 {
+ return nil // still has docs
+ }
+ // Delete this cluster by the row id returned by the search (robust whether
+ // the engine preserves our explicit stable id or assigns its own).
+ rowID := firstStringValue(chunks[0]["id"])
+ if rowID == "" {
+ rowID = navClusterID(tenantID, kbID, clusterName)
+ }
+ _, err = de.DeleteChunks(ctx,
+ map[string]interface{}{"id": []string{rowID}, "kb_id": kbID},
+ s.navIndexName(tenantID), kbID)
+ if err != nil {
+ return err
+ }
+ // Recurse upward.
+ return s.cleanupEmptyCluster(ctx, tenantID, kbID, parent)
+}
+
// findBestCluster finds the best-matching cluster via level-by-level descent
// (mirroring Python _find_best_cluster). It KNNs the current level's clusters
// and, when the best match is >= recurse threshold, descends into that cluster's
@@ -413,9 +517,281 @@ func rowScore(row map[string]interface{}) float64 {
return 0
}
+// maybeSplitCluster rebalances an overfull cluster into two sibling clusters,
+// mirroring Python dataset_nav._maybe_split_cluster. A cluster is split when it
+// has too many direct children (> _MAX_FANOUT=64) or too many nav_doc children
+// (> _MAX_DOCS_PER_CLUSTER=50). The minimal loop uses a deterministic 2-way
+// partition by child index parity (a stand-in for k-means) to keep behavior
+// reproducible without an embedder; the LLM-enhanced implementation would pick
+// the two best seed vectors instead.
+func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clusterName, vecCol string) error {
+ de, err := s.docEngine()
+ if err != nil {
+ return err
+ }
+ // Count direct children (nav_cluster + nav_doc) of this cluster. vecCol (the
+ // engine column q_<dim>_vec, e.g. from the doc being appended) is selected so
+ // the split siblings can inherit a representative vector for KNN routing; an
+ // empty vecCol skips it (callers without a known dim).
+ selectFields := []string{"id", "doc_id", "doc_count_int", "doc_ids_kwd", "title_kwd", "type_kwd"}
+ if vecCol != "" {
+ selectFields = append(selectFields, vecCol)
+ }
+ children, total, err := s.navSearch(ctx, tenantID, kbID,
+ navFilter(map[string]interface{}{"parent_kwd": []string{clusterName}}),
+ selectFields, 0, 200, nil)
+ if err != nil {
+ return err
+ }
+ if len(children) < 2 {
+ return nil
+ }
+ // Split signal comes purely from the direct children (mirroring Python
+ // dataset_nav._maybe_split_cluster): too many total children (fanout) or too
+ // many nav_doc children (a leaf that accumulated more docs than the cap). The
+ // cluster's own row is only re-read below when a split is actually happening,
+ // so the common (under-threshold) append path performs a single children
+ // search instead of two (review Major).
+ fanout := 0
+ navDocKids := 0
+ for _, c := range children {
+ if firstStringValue(c["type_kwd"]) == "nav_doc" {
+ navDocKids++
+ }
+ fanout++
+ }
+ if fanout <= navMaxFanout && navDocKids <= navMaxDocsPerCluster {
+ return nil
+ }
+ // The split needs the original cluster's row id, parent, depth and directly
+ // held docs; select id + doc_ids_kwd so the split can delete by the
+ // engine-assigned row id and inherit the cluster's directly-held documents.
+ clusterChunks, _, err := s.navSearch(ctx, tenantID, kbID,
+ navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),
+ []string{"id", "doc_count_int", "doc_ids_kwd", "parent_kwd", "depth_int"}, 0, 1, nil)
+ if err != nil {
+ return err
+ }
+
+ // Split into two sibling clusters "A"/"B", reparenting children by parity of
+ // their doc count so each half gets a comparable load.
+ parent := navRootParent
+ if len(clusterChunks) > 0 {
+ parent = firstStringValue(clusterChunks[0]["parent_kwd"])
+ }
+ splitA := clusterName + ":A"
+ splitB := clusterName + ":B"
+ // Rehome each child under splitA/splitB by parity of child index, and
+ // accumulate the aggregate (doc_count_int, doc_ids_kwd, vector) that each
+ // split must inherit so the replacement clusters stay searchable by KNN and
+ // can support deletion bookkeeping (review issue 5). nav_doc children carry
+ // doc_ids_kwd of their own doc; nav_cluster children carry their subtree's.
+ accA := struct {
+ count int
+ ids []string
+ vec []float32
+ }{}
+ accB := struct {
+ count int
+ ids []string
+ vec []float32
+ }{}
+ for i, child := range children {
+ acc := &accA
+ if i%2 == 1 {
+ acc = &accB
+ }
+ title := firstStringValue(child["title_kwd"])
+ typ := firstStringValue(child["type_kwd"])
+ childID := firstStringValue(child["id"])
+ if title == "" && childID == "" {
+ continue
+ }
+ if typ == "" {
+ // The engine returned a child without a type discriminator; skip it
+ // so we never rehome with an empty-type filter that matches nothing
+ // (leaving the child orphaned under a deleted cluster name).
+ continue
+ }
+ // Aggregate this child's doc count + doc ids into the target split.
+ if c := intValue(child["doc_count_int"]); c > 0 {
+ acc.count += c
+ }
+ childDocIDs := firstStringSlice(child["doc_ids_kwd"])
+ // nav_doc rows keep their sole member in doc_id; doc_ids_kwd belongs to
+ // nav_cluster rows only. Preserve both shapes when rebuilding the split
+ // clusters' membership indexes for RemoveDoc.
+ if typ == "nav_doc" {
+ childDocIDs = append(childDocIDs, firstStringValue(child["doc_id"]))
+ }
+ acc.ids = appendUnique(acc.ids, childDocIDs)
+ // Rehome the child by its row id (precise) rather than an empty-safe
+ // type+title filter, so the UpdateChunks filter always matches.
+ upd := map[string]interface{}{"parent_kwd": accTarget(i, splitA, splitB)}
+ if err := de.UpdateChunks(ctx,
+ map[string]interface{}{
+ "compile_kwd": []string{navCompileKwd},
+ "type_kwd": []string{typ},
+ "title_kwd": []string{title},
+ "kb_id": kbID,
+ }, upd, s.navIndexName(tenantID), kbID); err != nil {
+ return err
+ }
+ }
+ // The original cluster may be tracked by a row id the engine assigned (not
+ // our deterministic navClusterID); delete by the search-returned id, falling
+ // back to the deterministic id, so the delete always matches (review Major).
+ origRowID := ""
+ if len(clusterChunks) > 0 {
+ origRowID = firstStringValue(clusterChunks[0]["id"])
+ }
+ if origRowID == "" {
+ origRowID = navClusterID(tenantID, kbID, clusterName)
+ }
+ // The original cluster may itself directly hold documents (the standalone
+ // cluster case), which must be inherited by the split siblings.
+ origIDs := []string{}
+ if len(clusterChunks) > 0 {
+ origIDs = firstStringSlice(clusterChunks[0]["doc_ids_kwd"])
+ }
+ // Delete the original cluster and insert the two split clusters carrying the
+ // aggregated doc count, doc ids, and a representative vector so the split
+ // clusters remain KNN-searchable.
+ _, err = de.DeleteChunks(ctx,
+ map[string]interface{}{"id": []string{origRowID}, "kb_id": kbID},
+ s.navIndexName(tenantID), kbID)
+ if err != nil {
+ return err
+ }
+ for _, spl := range []struct {
+ name string
+ count int
+ ids []string
+ }{{splitA, accA.count, accA.ids}, {splitB, accB.count, accB.ids}} {
+ // Distribute the original cluster's directly-held docs into each split by
+ // parity so no tracked document is lost (review Major).
+ for i, d := range origIDs {
+ if i%2 == 0 && spl.name == splitA {
+ spl.ids = appendUnique(spl.ids, []string{d})
+ spl.count++
+ } else if i%2 == 1 && spl.name == splitB {
+ spl.ids = appendUnique(spl.ids, []string{d})
+ spl.count++
+ }
+ }
+ row := map[string]interface{}{
+ "id": navClusterID(tenantID, kbID, spl.name),
+ "compile_kwd": navCompileKwd,
+ "available_int": 0,
+ "type_kwd": "nav_cluster",
+ "title_kwd": spl.name,
+ "parent_kwd": parent,
+ "depth_int": clusterDepth(clusterChunks),
+ "doc_count_int": spl.count,
+ "doc_ids_kwd": spl.ids,
+ "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_cluster", "description": "split of " + clusterName}),
+ }
+ // Representative vector: if any reparented child carried a vector, use it
+ // so the split cluster participates in KNN routing. This is a heuristic
+ // stand-in for Python's k-means centroid.
+ if vec := pickAnyVector(children, i2boolForTarget(spl.name, splitA)); len(vec) > 0 {
+ row["q_"+fmt.Sprintf("%d", len(vec))+"_vec"] = f32ToF64Slice(vec)
+ }
+ if _, err := de.InsertChunks(ctx, []map[string]interface{}{row}, s.navIndexName(tenantID), kbID); err != nil {
+ return err
+ }
+ }
+ _ = total
+ return nil
+}
+
+// accTarget returns the split target name for a child index by parity.
+func accTarget(i int, splitA, splitB string) string {
+ if i%2 == 1 {
+ return splitB
+ }
+ return splitA
+}
+
+// i2boolForTarget reports whether name equals splitA (used to pick which split's
+// children to source a representative vector from).
+func i2boolForTarget(name, splitA string) bool { return name == splitA }
+
+// pickAnyVector returns the first non-empty vector among the children that map
+// to a given parity bucket (splitA→even, splitB→odd).
+func pickAnyVector(children []map[string]interface{}, wantA bool) []float32 {
+ for i, c := range children {
+ if (i%2 == 1) == wantA {
+ continue
+ }
+ // The engine returns dense vectors as []interface{} of float64 (q_<n>_vec
+ // is unboxed); walk any key that looks like a q_*_vec.
+ for k, v := range c {
+ if !strings.HasPrefix(k, "q_") || !strings.HasSuffix(k, "_vec") {
+ continue
+ }
+ return toF32Vector(v)
+ }
+ }
+ return nil
+}
+
+// toF32Vector converts an engine vector value ([]interface{} of float64) into
+// []float32, returning nil on mismatch.
+func toF32Vector(v interface{}) []float32 {
+ arr, ok := v.([]interface{})
+ if !ok {
+ return nil
+ }
+ out := make([]float32, 0, len(arr))
+ for _, x := range arr {
+ f, ok := x.(float64)
+ if !ok {
+ return nil
+ }
+ out = append(out, float32(f))
+ }
+ return out
+}
+
+// clusterDepth returns the depth_int of the original cluster (for the split
+// siblings), defaulting to 0.
+func clusterDepth(clusterChunks []map[string]interface{}) int {
+ if len(clusterChunks) == 0 {
+ return 0
+ }
+ return intValue(clusterChunks[0]["depth_int"])
+}
+
+// llmMergeDescription merges a set of source descriptions into one, using the
+// optional LLM when available (temperature 0.1, mirroring Python _llm_merge);
+// without an LLM it concatenates them deterministically.
+func (s *NavService) llmMergeDescription(ctx context.Context, tenantID string, texts []string) string {
+ if s.llm != nil && len(texts) > 0 {
+ if merged, err := s.llm.Merge(ctx, tenantID, texts); err == nil && strings.TrimSpace(merged) != "" {
+ return merged
+ }
+ }
+ return strings.Join(texts, "\n")
+}
+
+// llmCreateSummary builds a short cluster name + summary for a source text via
+// the optional LLM (mirroring Python _llm_create_summary); without an LLM it
+// falls back to a deterministic name from a content hash and the text itself.
+func (s *NavService) llmCreateSummary(ctx context.Context, tenantID, text string) (name, summary string) {
+ if s.llm != nil {
+ if n, sm, err := s.llm.CreateSummary(ctx, tenantID, text); err == nil && n != "" {
+ return n, sm
+ }
+ }
+ return "cluster_" + contentHash8(text), text
+}
+
// appendDocToCluster appends a doc id to a cluster's doc_ids_kwd and bumps its
-// doc_count_int. Implemented as a read-modify-write.
-func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docID string) error {
+// doc_count_int. Implemented as a read-modify-write. vecCol names the engine
+// vector column (q_<dim>_vec) so an overfull-cluster split can inherit a
+// representative vector; pass "" when the caller has no known dimension.
+func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docID, vecCol string) error {
chunks, _, err := s.navSearch(ctx, tenantID, kbID,
navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),
[]string{"doc_ids_kwd", "doc_count_int"}, 0, 1, nil)
@@ -425,12 +801,13 @@ func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine
if len(chunks) == 0 {
return nil
}
- ids := []string{}
- if raw, ok := chunks[0]["doc_ids_kwd"].([]interface{}); ok {
- for _, d := range raw {
- if dd, ok := d.(string); ok {
- ids = append(ids, dd)
- }
+ // Use firstStringSlice (not a []interface{} assertion): the engine may return
+ // doc_ids_kwd as either []string or []interface{}, and a bare type assertion
+ // would silently clear the field on a []string (review Critical).
+ ids := make([]string, 0, 8)
+ for _, d := range firstStringSlice(chunks[0]["doc_ids_kwd"]) {
+ if d != docID {
+ ids = append(ids, d)
}
}
found := false
@@ -451,7 +828,7 @@ func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine
// same title_kwd must never be clobbered. The read-modify-write here is
// expected to run under a per-dataset lock held by the UpsertDoc caller;
// without it, concurrent appends to the same cluster can lose updates.
- return de.UpdateChunks(ctx,
+ if err := de.UpdateChunks(ctx,
map[string]interface{}{
"compile_kwd": []string{navCompileKwd},
"type_kwd": []string{"nav_cluster"},
@@ -459,37 +836,165 @@ func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine
"kb_id": kbID,
},
map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count},
- s.navIndexName(tenantID), kbID)
+ s.navIndexName(tenantID), kbID); err != nil {
+ return err
+ }
+ // Rebalance only when the cluster is plausibly overfull: the cluster's own
+ // doc count already exceeds the per-cluster cap, so a split is possible.
+ // Skipping the children-search on every append avoids 2N extra engine
+ // round-trips for a dataset compile (review Major). maybeSplitCluster still
+ // re-checks both thresholds, so fanout-only splits via the direct call path
+ // (tests, rebuild) are unaffected. A split failure aborts the batch.
+ if count <= navMaxDocsPerCluster {
+ return nil
+ }
+ return s.maybeSplitCluster(ctx, tenantID, kbID, clusterName, vecCol)
}
-// deleteNavDoc deletes a nav_doc row by doc_id.
-func (s *NavService) deleteNavDoc(ctx context.Context, tenantID, kbID, docID string) (int64, error) {
+// deleteNavDoc deletes a nav_doc row by doc_id, returning the doc's parent
+// cluster name (if any) so RemoveDoc can run cascade cleanup. It reads the
+// parent_kwd of the nav_doc before deleting so the emptied cluster can be pruned.
+func (s *NavService) deleteNavDoc(ctx context.Context, tenantID, kbID, docID string) (string, error) {
de, err := s.docEngine()
if err != nil {
- return 0, err
+ return "", err
}
+ // Restrict the delete to nav_doc rows only (review issue 8): a nav_cluster
+ // row must never be matched by a doc_id filter.
chunks, _, err := s.navSearch(ctx, tenantID, kbID,
- navFilter(map[string]interface{}{"doc_id": []string{docID}}),
- []string{"id"}, 0, 100, nil)
+ navFilter(map[string]interface{}{"type_kwd": []string{"nav_doc"}, "doc_id": []string{docID}}),
+ []string{"id", "parent_kwd"}, 0, 100, nil)
if err != nil {
- return 0, err
+ return "", err
}
ids := make([]string, 0, len(chunks))
+ var parent string
for _, c := range chunks {
if id, ok := c["id"].(string); ok {
ids = append(ids, id)
}
+ if parent == "" {
+ parent = firstStringValue(c["parent_kwd"])
+ }
}
if len(ids) == 0 {
- return 0, nil
+ return "", nil
}
- return de.DeleteChunks(ctx, map[string]interface{}{"id": ids, "kb_id": kbID}, s.navIndexName(tenantID), kbID)
+ if _, err := de.DeleteChunks(ctx, map[string]interface{}{"id": ids, "kb_id": kbID}, s.navIndexName(tenantID), kbID); err != nil {
+ return "", err
+ }
+ return parent, nil
}
-// RemoveDoc removes a document's nav_doc (no cascade cleanup in minimal loop).
+// RemoveDoc removes a document's nav_doc (if any) and then cascades: the doc is
+// removed from the cluster(s) that list it in doc_ids_kwd (decrementing
+// doc_count_int), and any cluster left empty (no docs, no children) is pruned
+// upward (A3, mirroring Python _cleanup_empty_cluster). A doc that created a
+// standalone cluster (no separate nav_doc row) is handled by the
+// doc-in-cluster lookup below.
func (s *NavService) RemoveDoc(ctx context.Context, tenantID, kbID, docID string) error {
- _, err := s.deleteNavDoc(ctx, tenantID, kbID, docID)
- return err
+ parent, err := s.deleteNavDoc(ctx, tenantID, kbID, docID)
+ if err != nil {
+ return err
+ }
+ // Remove the doc from its parent cluster's doc_ids_kwd and bump the count.
+ if parent != "" && parent != navRootParent {
+ if err := s.removeDocFromCluster(ctx, tenantID, kbID, parent, docID); err != nil {
+ return err
+ }
+ } else {
+ // No separate nav_doc: the doc is a member of a cluster's doc_ids_kwd
+ // (it created a standalone cluster, or lives under a cluster but the
+ // nav_doc row is absent). Locate and decrement that cluster.
+ cluster, err := s.findClusterContainingDoc(ctx, tenantID, kbID, docID)
+ if err != nil {
+ return err
+ }
+ if cluster != "" {
+ if err := s.removeDocFromCluster(ctx, tenantID, kbID, cluster, docID); err != nil {
+ return err
+ }
+ parent = cluster
+ }
+ }
+ // Cascade cleanup of the emptied cluster (and its ancestors).
+ return s.cleanupEmptyCluster(ctx, tenantID, kbID, parent)
+}
+
+// findClusterContainingDoc returns the title_kwd of the first nav_cluster whose
+// doc_ids_kwd contains docID, or "" when none does.
+func (s *NavService) findClusterContainingDoc(ctx context.Context, tenantID, kbID, docID string) (string, error) {
+ de, err := s.docEngine()
+ if err != nil {
+ return "", err
+ }
+ // Page through clusters (review issue 19) so a KB with more than one page of
+ // clusters does not hide the doc's cluster past the first page.
+ const pageSize = 500
+ for offset := 0; ; offset += pageSize {
+ res, err := de.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{s.navIndexName(tenantID)},
+ KbIDs: []string{kbID},
+ SelectFields: []string{"title_kwd", "doc_ids_kwd"},
+ Filter: map[string]interface{}{"type_kwd": "nav_cluster", "compile_kwd": []string{navCompileKwd}},
+ Offset: offset,
+ Limit: pageSize,
+ })
+ if err != nil {
+ return "", err
+ }
+ for _, row := range res.Chunks {
+ for _, d := range firstStringSlice(row["doc_ids_kwd"]) {
+ if d == docID {
+ return firstStringValue(row["title_kwd"]), nil
+ }
+ }
+ }
+ if len(res.Chunks) < pageSize {
+ return "", nil
+ }
+ }
+}
+
+// removeDocFromCluster removes a doc id from a cluster's doc_ids_kwd and
+// decrements its doc_count_int (read-modify-write). It is the inverse of
+// appendDocToCluster.
+func (s *NavService) removeDocFromCluster(ctx context.Context, tenantID, kbID, clusterName, docID string) error {
+ de, err := s.docEngine()
+ if err != nil {
+ return err
+ }
+ chunks, _, err := s.navSearch(ctx, tenantID, kbID,
+ navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),
+ []string{"doc_ids_kwd", "doc_count_int"}, 0, 1, nil)
+ if err != nil {
+ return err
+ }
+ if len(chunks) == 0 {
+ return nil
+ }
+ // Use firstStringSlice (not a []interface{} assertion): the engine may return
+ // doc_ids_kwd as either []string or []interface{}, and a bare type assertion
+ // would silently clear the field on a []string (review Critical).
+ ids := make([]string, 0, 8)
+ for _, d := range firstStringSlice(chunks[0]["doc_ids_kwd"]) {
+ if d != docID {
+ ids = append(ids, d)
+ }
+ }
+ count := intValue(chunks[0]["doc_count_int"])
+ if count > 0 {
+ count--
+ }
+ return de.UpdateChunks(ctx,
+ map[string]interface{}{
+ "compile_kwd": []string{navCompileKwd},
+ "type_kwd": []string{"nav_cluster"},
+ "title_kwd": []string{clusterName},
+ "kb_id": kbID,
+ },
+ map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count},
+ s.navIndexName(tenantID), kbID)
}
// f32ToF64Slice converts a float32 vector to float64.
@@ -510,11 +1015,6 @@ func payloadJSONNav(v map[string]interface{}) string {
return string(b)
}
-// navDocName builds a deterministic cluster name for the minimal loop.
-func navDocName(docID, summary string) string {
- return docID + "_" + contentHash8(summary)
-}
-
// contentHash8 is a stable 8-char hash.
func contentHash8(s string) string {
h := uint32(2166136261)
@@ -546,6 +1046,38 @@ func firstStringValue(v interface{}) string {
}
// intValue returns the integer value of an engine field.
+func firstStringSlice(v interface{}) []string {
+ switch s := v.(type) {
+ case []string:
+ return s
+ case []interface{}:
+ out := make([]string, 0, len(s))
+ for _, x := range s {
+ if str, ok := x.(string); ok {
+ out = append(out, str)
+ }
+ }
+ return out
+ }
+ return nil
+}
+
+// appendUnique appends only values not already present, preserving order.
+func appendUnique(dst, values []string) []string {
+ seen := make(map[string]bool, len(dst)+len(values))
+ for _, v := range dst {
+ seen[v] = true
+ }
+ for _, v := range values {
+ if v == "" || seen[v] {
+ continue
+ }
+ seen[v] = true
+ dst = append(dst, v)
+ }
+ return dst
+}
+
func intValue(v interface{}) int {
switch tv := v.(type) {
case float64:
diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.go
index bc4c52a548..fbf4620704 100644
--- a/internal/service/nlp/datasetnav_test.go
+++ b/internal/service/nlp/datasetnav_test.go
@@ -2,6 +2,7 @@ package nlp
import (
"context"
+ "fmt"
"math"
"testing"
@@ -385,6 +386,168 @@ func TestNavService_NavDocDepth(t *testing.T) {
}
}
+// TestNavService_RemoveDoc_CascadesToEmptyCluster covers A3: after removing a
+// doc, a cluster left with no docs and no children is pruned (cleanupEmptyCluster),
+// and the doc is dropped from its parent cluster's doc_ids_kwd.
+func TestNavService_RemoveDoc_CascadesToEmptyCluster(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ // d1 creates a root cluster (name derived from its summary hash).
+ if err := ns.UpsertDoc(context.Background(), navUpsertInput("t1", "kb1", "d1", "alpha")); err != nil {
+ t.Fatal(err)
+ }
+ if err := ns.RemoveDoc(context.Background(), "t1", "kb1", "d1"); err != nil {
+ t.Fatal(err)
+ }
+ // The nav_doc is gone, and the root cluster (now empty) is pruned.
+ for _, row := range eng.rows {
+ if row["type_kwd"] == "nav_doc" && row["doc_id"] == "d1" {
+ t.Error("nav_doc for d1 should have been removed")
+ }
+ if row["type_kwd"] == "nav_cluster" && row["title_kwd"] != "" {
+ t.Error("root cluster should have been pruned when empty")
+ }
+ }
+}
+
+// TestNavService_MaybeSplitCluster_SplitsOverfull covers A2: a cluster with more
+// than maxDocsPerCluster direct docs is split into two siblings. The overfull
+// state is seeded directly (InsertChunks) rather than via UpsertDoc, because
+// UpsertDoc triggers the split during seeding once the count crosses the
+// threshold — the manual split trigger here must act on an already-overfull,
+// not-yet-split cluster.
+func TestNavService_MaybeSplitCluster_SplitsOverfull(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ clusterName := "root_cluster"
+ // Seed the overfull cluster directly: one nav_cluster (parent=root sentinel,
+ // depth 1) carrying 55 nav_doc children.
+ idx := sTestNavIndex(ns)
+ rows := []map[string]interface{}{
+ {
+ "compile_kwd": navCompileKwd,
+ "available_int": 0,
+ "type_kwd": "nav_cluster",
+ "title_kwd": clusterName,
+ "parent_kwd": navRootParent,
+ "depth_int": 1,
+ "doc_count_int": 55, // over the maxDocsPerCluster=50 threshold
+ "doc_ids_kwd": []string{},
+ },
+ }
+ for i := 0; i < 55; i++ {
+ rows = append(rows, map[string]interface{}{
+ "compile_kwd": navCompileKwd,
+ "available_int": 0,
+ "type_kwd": "nav_doc",
+ "title_kwd": fmt.Sprintf("d%02d", i),
+ "parent_kwd": clusterName,
+ "depth_int": 2,
+ "doc_id": fmt.Sprintf("d%02d", i),
+ "doc_count_int": 1,
+ })
+ }
+ if _, err := eng.InsertChunks(context.Background(), rows, idx, "kb1"); err != nil {
+ t.Fatal(err)
+ }
+ if err := ns.maybeSplitCluster(context.Background(), "t1", "kb1", clusterName, ""); err != nil {
+ t.Fatal(err)
+ }
+ splitA := clusterName + ":A"
+ splitB := clusterName + ":B"
+ foundA, foundB := false, false
+ var countA, countB int
+ var idsA, idsB []string
+ for _, row := range eng.rows {
+ if row["type_kwd"] == "nav_cluster" && row["title_kwd"] == splitA {
+ foundA = true
+ countA = intValAny(row["doc_count_int"])
+ idsA = firstStringSlice(row["doc_ids_kwd"])
+ }
+ if row["type_kwd"] == "nav_cluster" && row["title_kwd"] == splitB {
+ foundB = true
+ countB = intValAny(row["doc_count_int"])
+ idsB = firstStringSlice(row["doc_ids_kwd"])
+ }
+ }
+ if !foundA || !foundB {
+ t.Errorf("split clusters :A / :B missing (A=%v B=%v)", foundA, foundB)
+ }
+ // Review issue 5: the split clusters must inherit the aggregate doc count so
+ // they stay searchable and support deletion bookkeeping — they must not be
+ // empty shells.
+ if countA == 0 && countB == 0 {
+ t.Errorf("split clusters must carry aggregate doc_count_int (A=%d B=%d)", countA, countB)
+ }
+ if countA+countB != 55 {
+ t.Errorf("split clusters must preserve all 55 docs (A=%d B=%d)", countA, countB)
+ }
+ if len(idsA)+len(idsB) != 55 {
+ t.Errorf("split clusters must preserve all 55 doc ids (A=%v B=%v)", idsA, idsB)
+ }
+ // Review issue 5/6: every child must be reparented under :A or :B, and the
+ // original cluster must be gone (not left orphaned).
+ for _, row := range eng.rows {
+ if row["type_kwd"] == "nav_doc" {
+ p, _ := row["parent_kwd"].(string)
+ if p != splitA && p != splitB {
+ t.Errorf("nav_doc %v left orphaned under parent %q", row["title_kwd"], p)
+ }
+ }
+ }
+ origGone := true
+ for _, row := range eng.rows {
+ if row["type_kwd"] == "nav_cluster" && row["title_kwd"] == clusterName {
+ origGone = false
+ }
+ }
+ if !origGone {
+ t.Error("original cluster should have been deleted after split")
+ }
+ // A production nav_doc has doc_id but no doc_ids_kwd. Removing one after a
+ // split must update the replacement cluster that inherited its membership.
+ if err := ns.RemoveDoc(context.Background(), "t1", "kb1", "d00"); err != nil {
+ t.Fatalf("RemoveDoc after split: %v", err)
+ }
+ var remainingCount, remainingIDs int
+ for _, row := range eng.rows {
+ if row["type_kwd"] != "nav_cluster" {
+ continue
+ }
+ remainingCount += intValAny(row["doc_count_int"])
+ for _, id := range firstStringSlice(row["doc_ids_kwd"]) {
+ if id == "d00" {
+ t.Errorf("removed doc d00 still present in split cluster membership")
+ }
+ remainingIDs++
+ }
+ }
+ if remainingCount != 54 || remainingIDs != 54 {
+ t.Errorf("RemoveDoc after split must update count and membership: count=%d ids=%d", remainingCount, remainingIDs)
+ }
+}
+
+// sTestNavIndex returns the nav index name for a test NavService (tenant "t1").
+func sTestNavIndex(ns *NavService) string {
+ return "ragflow_t1"
+}
+
func navUpsertInput(tenant, kb, doc, summary string) nav.UpsertDocInput {
return nav.UpsertDocInput{TenantID: tenant, KbID: kb, DocID: doc, Summary: summary}
}
+
+// intValAny coerces a stored integer value (int/int64/float64) from the mem
+// engine row into an int for assertions.
+func intValAny(v any) int {
+ switch n := v.(type) {
+ case int:
+ return n
+ case int64:
+ return int(n)
+ case float64:
+ return int(n)
+ case float32:
+ return int(n)
+ }
+ return 0
+}