Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(nav): navigation tree search, LLM cluster naming, cluster depth (#20188)

qinling0210ab247dc50894
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index 29613c56a5..49bb9dfda5 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -818,7 +818,16 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, serverName stri
// unavailable, skipping dataset-nav upsert"), leaving the dataset tree empty.
// The embedder resolves the tenant's embedding model on demand, so both
// Search and UpsertDoc can embed queries/summaries automatically.
- nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(service.NewModelProviderService(), "")))
+ navModelService := service.NewModelProviderService()
+ navService := nlp.NewNavService(service.NewNavEmbedder(navModelService, ""))
+ // The cluster namer/merger is the counterpart of Python's
+ // _llm_create_summary / _llm_merge: a new cluster is named after the model's
+ // topic title and an existing cluster's description is fused with the joining
+ // document. Passing an empty model ref resolves each tenant's default chat
+ // model on demand; without one the nav service keeps its deterministic
+ // behaviour.
+ navService.SetNavMergeLLM(service.NewNavMergeLLM(navModelService, ""))
+ nav.SetNavService(navService)
// Memory extraction runs on the Ingestor's shared NATS consumer + worker
// pool (task_type="memory" dispatched by handleAndExecute -> executeMemoryTask),
// so there is no longer a dedicated Redis memory consumer to start.
@@ -1141,8 +1150,12 @@ func startServer(ctx context.Context, serverName string, arguments *serverArgs)
// Install the dataset-nav ES-backed service (internal/service/nav +
// internal/service/nlp). The embedder resolves the tenant's embedding model
- // on demand so Search/UpsertDoc can embed queries/summaries automatically.
- nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(modelProviderService, "")))
+ // on demand so Search/UpsertDoc can embed queries/summaries automatically,
+ // and the cluster namer/merger resolves the tenant's default chat model the
+ // same way (see the ingestor's wrapping of SetNavMergeLLM).
+ apiNavService := nlp.NewNavService(service.NewNavEmbedder(modelProviderService, ""))
+ apiNavService.SetNavMergeLLM(service.NewNavMergeLLM(modelProviderService, ""))
+ nav.SetNavService(apiNavService)
// Install the compiled-wiki search service. It is backed directly by the
// document engine: QueryPages filters the tenant-scoped index to
diff --git a/internal/service/nav/nav.go b/internal/service/nav/nav.go
index ff1405cf86..c1d77d658f 100644
--- a/internal/service/nav/nav.go
+++ b/internal/service/nav/nav.go
@@ -37,6 +37,14 @@ type NavNode struct {
Type string `json:"type"`
DocID string `json:"doc_id,omitempty"`
HasChildren bool `json:"has_children"`
+ // Parent is the tree edge: the parent cluster's name, or "root" for a
+ // depth-0 cluster. A search result returns matched leaves TOGETHER with the
+ // cluster path above them, so the caller nests the flat list into one tree
+ // per root cluster instead of rendering every hit as its own tree.
+ Parent string `json:"parent_kwd,omitempty"`
+ // Matched marks a keyword-search HIT; the path rows returned alongside it are
+ // structure only. Rows outside a search never set it.
+ Matched bool `json:"matched,omitempty"`
}
// Nav row types as stored in the compiled nav index (type_kwd).
@@ -69,8 +77,11 @@ type UpsertDocInput struct {
// production chat invoker without an import cycle; nil disables LLM behavior and
// the implementation falls back to deterministic naming.
type NavMergeLLM interface {
- // Merge returns a merged cluster description for one or more source texts
- // (temperature 0.1, mirroring Python _llm_merge).
+ // Merge fuses a new document summary into a cluster's description
+ // (temperature 0.1, mirroring Python _llm_merge). texts is
+ // [existing description, new document summary]; on failure it must return the
+ // existing description (an error also keeps it), so a model hiccup never
+ // blanks a stored description.
Merge(ctx context.Context, tenantID string, texts []string) (string, error)
// CreateSummary returns a short cluster name + summary for a source text
// (mirroring Python _llm_create_summary). Return name="" to fall back.
@@ -96,17 +107,22 @@ type NavService interface {
// returned coverage is trimmed to the scope so no out-of-scope document
// surfaces under a cluster that merely overlaps it.
Search(ctx context.Context, tenantID, kbID, query string, embd []float32, docScope []string, topK int) ([]NavHit, error)
- // ListClusters returns the depth-0 clusters (parent_kwd=root). When
- // keywords is non-empty, it returns matching navigation nodes instead.
+ // ListClusters returns the depth-0 clusters (parent_kwd=root), or — when
+ // keywords is non-empty — the pruned search forest: the matched leaves plus
+ // the cluster path above each of them. The total is a cluster count in both
+ // modes.
ListClusters(ctx context.Context, tenantID, kbID, keywords string, page, pageSize int) ([]NavNode, int64, error)
// ListChildren returns the direct children of a cluster (parent_kwd=name),
// optionally filtered by keywords.
ListChildren(ctx context.Context, tenantID, kbID, name, keywords string, page, pageSize int) ([]NavNode, int64, error)
- // SummariesByDocIDs returns the nav_doc summary (preferred readable name, else
- // the payload description) keyed by doc_id for the given documents. It mirrors
- // Python dataset_api_service._nav_doc_summaries and backs the chunk_agg
- // navigation-tree router's document labels. A doc without a nav_doc row is
- // absent from the result.
+ // SummariesByDocIDs returns the nav_doc description — the document's overall
+ // summary, as stored in the row's payload — keyed by doc_id for the given
+ // documents, falling back to the row's readable label when the payload
+ // carries no description. It mirrors Python
+ // dataset_api_service._nav_doc_summaries and backs the chunk_agg
+ // navigation-tree router's document labels, which callers render to the model
+ // as the document's <summary>. A doc without a nav_doc row is absent from the
+ // result.
SummariesByDocIDs(ctx context.Context, tenantID, kbID string, docIDs []string) map[string]string
}
diff --git a/internal/service/nav_merge_llm.go b/internal/service/nav_merge_llm.go
new file mode 100644
index 0000000000..afd497db14
--- /dev/null
+++ b/internal/service/nav_merge_llm.go
@@ -0,0 +1,258 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "strings"
+
+ "ragflow/internal/entity"
+ modelModule "ragflow/internal/entity/models"
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+)
+
+// navNamingTemperature is the decoding temperature Python pins for both cluster
+// prompts (_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1})).
+const navNamingTemperature = 0.1
+
+// navNamingMaxRetries mirrors Python gen_json's max_retry=2 for the JSON naming
+// prompt: each retry re-issues the call with the parse error appended, so a
+// formatting hiccup costs two calls, not the compiler's default five.
+const navNamingMaxRetries = 2
+
+// NavMergeLLM is the production implementation of the nav.NavMergeLLM seam: it
+// names dataset-nav clusters and fuses cluster descriptions with the tenant's
+// chat model, mirroring Python dataset_nav._llm_create_summary / _llm_merge.
+//
+// It sits next to NavEmbedder in the service package, not with the seam it
+// implements, because it needs ModelProviderService and no inner package may
+// import service: service already imports nlp (chat_pipeline, deep_researcher,
+// memory) and nav (dataset_artifact_service). The nav package is deliberately a
+// dependency-light leaf — the agent tool layer and the nlp implementation both
+// depend on it — so only the seam lives there and the model-backed adapters stay
+// out here. Moving this type into nav would need the model access reduced to an
+// injected interface first.
+type NavMergeLLM struct {
+ modelSvc *ModelProviderService
+ // llmID is the configured chat model ref (a composite
+ // "model@instance@provider" name or a tenant model id). Empty resolves the
+ // tenant's default chat model, matching the dataset-level deduper's fallback.
+ llmID string
+}
+
+// NewNavMergeLLM builds the production nav cluster namer/merger.
+func NewNavMergeLLM(modelSvc *ModelProviderService, llmID string) *NavMergeLLM {
+ return &NavMergeLLM{modelSvc: modelSvc, llmID: llmID}
+}
+
+// navNamingPrompt is the cluster-naming prompt (Python _llm_create_summary) plus
+// a language constraint. The prompt is written in English, so the model answered
+// in English even for a Chinese dataset, and the cluster name it produced (with
+// the deterministic hash suffix appended) is what the nav tree displays — an
+// English label over Chinese documents. The excerpts' own language is the one
+// the tree's readers use, so it wins.
+func navNamingPrompt(text string) string {
+ return "Given the document excerpts below, produce a short human-readable topic " +
+ "name and a concise description of their common topic.\n\n" +
+ text + "\n\n" +
+ "Write both the name and the description in the same language as the " +
+ "excerpts (do not translate them into another language).\n\n" +
+ `Return ONLY JSON: {"name": "<2-6 word topic title>", "summary": "<1-3 sentence description>"}`
+}
+
+// navMergePrompt fuses an existing cluster description with a new document
+// summary (Python _llm_merge). Same language rule, anchored on the NEW text: a
+// cluster whose description was written in another language converges back to
+// the corpus language as documents are merged in, instead of drifting.
+func navMergePrompt(existing, added string) string {
+ return "Merge the following two descriptions of the same topic into " +
+ "a single concise summary (1-3 sentences):\n\n" +
+ "Existing: " + existing + "\n\n" +
+ "New: " + added + "\n\n" +
+ "Write the merged summary in the same language as the New description " +
+ "(do not translate it).\n\n" +
+ "Return ONLY the merged text, no commentary."
+}
+
+// CreateSummary asks the model for a short topic name plus a 1-3 sentence
+// description of the supplied document text (Python _llm_create_summary). An
+// empty name or an error makes the caller fall back to the deterministic
+// "<summary first line> <hash>" name.
+func (l *NavMergeLLM) CreateSummary(ctx context.Context, tenantID, text string) (name, summary string, err error) {
+ text = strings.TrimSpace(text)
+ if text == "" {
+ return "", "", nil
+ }
+ llmID, err := l.chatModelID(ctx, tenantID)
+ if err != nil {
+ return "", "", err
+ }
+ temp := navNamingTemperature
+ reply, err := kccommon.GenJSON(ctx, navChatInvoker{modelSvc: l.modelSvc, tenantID: tenantID, llmID: llmID},
+ kccommon.ChatRequest{UserPrompt: navNamingPrompt(text), Temperature: &temp}, navNamingMaxRetries)
+ if err != nil {
+ return "", "", err
+ }
+ name, summary = navSummaryFromReply(reply, text)
+ return name, summary, nil
+}
+
+// Merge fuses the existing cluster description with a new document summary into
+// one concise summary (Python _llm_merge(chat_mdl, cluster_desc, doc_summary)).
+// texts is [existing description, new document summary]; the existing text is
+// returned unchanged when there is nothing to merge or the model reply is
+// unusable, so the caller keeps the stored description.
+func (l *NavMergeLLM) Merge(ctx context.Context, tenantID string, texts []string) (string, error) {
+ existing := ""
+ if len(texts) > 0 {
+ existing = strings.TrimSpace(texts[0])
+ }
+ added := ""
+ if len(texts) > 1 {
+ added = strings.TrimSpace(texts[1])
+ }
+ if added == "" {
+ return existing, nil
+ }
+ llmID, err := l.chatModelID(ctx, tenantID)
+ if err != nil {
+ return "", err
+ }
+ temp := navNamingTemperature
+ reply, err := navChatText(ctx, l.modelSvc, tenantID, llmID, "", navMergePrompt(existing, added), &temp)
+ if err != nil {
+ return "", err
+ }
+ return navMergedFromReply(reply, existing), nil
+}
+
+// chatModelID resolves the chat model ref to call: the configured one, else the
+// tenant's default chat model.
+func (l *NavMergeLLM) chatModelID(ctx context.Context, tenantID string) (string, error) {
+ if l.modelSvc == nil {
+ return "", fmt.Errorf("datasetnav: model provider service not initialized")
+ }
+ if ref := strings.TrimSpace(l.llmID); ref != "" {
+ return ref, nil
+ }
+ target, err := l.modelSvc.modelSolver().ResolveDefaultModelConfig(ctx, tenantID, entity.ModelTypeChat)
+ if err != nil {
+ return "", fmt.Errorf("datasetnav: resolve default chat model for tenant %s: %w", tenantID, err)
+ }
+ if target == nil || strings.TrimSpace(target.ModelID) == "" {
+ return "", fmt.Errorf("datasetnav: no default chat model for tenant %s", tenantID)
+ }
+ return target.ModelID, nil
+}
+
+// navSummaryFromReply mirrors Python _llm_create_summary's response handling:
+// name from "name", summary from "summary" then "result", the input text as the
+// last resort. An empty name is returned as-is so the caller falls back to its
+// deterministic name.
+func navSummaryFromReply(reply map[string]any, fallbackSummary string) (name, summary string) {
+ summary = navReplyString(reply["summary"])
+ if summary == "" {
+ summary = navReplyString(reply["result"])
+ }
+ if summary == "" {
+ summary = strings.TrimSpace(fallbackSummary)
+ }
+ return navReplyString(reply["name"]), summary
+}
+
+// navMergedFromReply mirrors Python _llm_merge's reply handling, with one
+// deliberate deviation: a plain-text reply IS used. Python feeds the reply
+// through gen_json, whose json_repair parse of ordinary prose yields an empty
+// string, so its `isinstance(resp, str) and resp.strip()` branch never fires and
+// the merge silently keeps the old description — even though the prompt asks for
+// "only the merged text". Keeping the reply body is what the prompt intends.
+func navMergedFromReply(reply, existing string) string {
+ reply = strings.TrimSpace(reply)
+ if reply == "" {
+ return existing
+ }
+ if repaired, err := kccommon.RepairJSONText(reply); err == nil {
+ var m map[string]any
+ if json.Unmarshal([]byte(repaired), &m) == nil {
+ if merged := navReplyString(m["merged"]); merged != "" {
+ return merged
+ }
+ if merged := navReplyString(m["result"]); merged != "" {
+ return merged
+ }
+ return existing
+ }
+ }
+ return reply
+}
+
+// navReplyString reads a string field out of an LLM JSON reply. Non-string
+// values are ignored: the prompts only ever ask for strings, and stringifying a
+// nested object would leak a debug repr into a stored description.
+func navReplyString(v any) string {
+ s, ok := v.(string)
+ if !ok {
+ return ""
+ }
+ return strings.TrimSpace(s)
+}
+
+// navChatInvoker adapts the model provider service to the knowledge-compiler
+// ChatInvoker seam, so the JSON naming prompt reuses kccommon.GenJSON's
+// extraction/repair/retry handling (the Go counterpart of Python gen_json).
+type navChatInvoker struct {
+ modelSvc *ModelProviderService
+ tenantID string
+ llmID string
+}
+
+func (c navChatInvoker) Chat(ctx context.Context, req kccommon.ChatRequest) (*kccommon.ChatResponse, error) {
+ content, err := navChatText(ctx, c.modelSvc, c.tenantID, c.llmID, req.SystemPrompt, req.UserPrompt, req.Temperature)
+ if err != nil {
+ return nil, err
+ }
+ return &kccommon.ChatResponse{Content: content}, nil
+}
+
+// navChatText dispatches one non-streaming chat call. system may be empty (both
+// nav prompts are single-turn, matching Python's gen_json("", prompt, ...)).
+func navChatText(ctx context.Context, modelSvc *ModelProviderService, tenantID, llmID, system, user string, temperature *float64) (string, error) {
+ if modelSvc == nil {
+ return "", fmt.Errorf("datasetnav: model provider service not initialized")
+ }
+ if strings.TrimSpace(llmID) == "" {
+ return "", fmt.Errorf("datasetnav: chat model is not resolved")
+ }
+ config := &modelModule.ChatConfig{}
+ if temperature != nil {
+ config.Temperature = temperature
+ }
+ msgs := []modelModule.Message{
+ {Role: "system", Content: system},
+ {Role: "user", Content: user},
+ }
+ resp, err := modelSvc.Chat(ctx, tenantID, llmID, msgs, config)
+ if err != nil {
+ return "", err
+ }
+ if resp == nil || resp.Answer == nil {
+ return "", nil
+ }
+ return strings.TrimSpace(*resp.Answer), nil
+}
diff --git a/internal/service/nav_merge_llm_test.go b/internal/service/nav_merge_llm_test.go
new file mode 100644
index 0000000000..a983913349
--- /dev/null
+++ b/internal/service/nav_merge_llm_test.go
@@ -0,0 +1,165 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "strings"
+ "testing"
+)
+
+// TestNavPromptsConstrainLanguage pins the language rule on both cluster prompts.
+// The prompts are English, and without the rule the model answered in English
+// even for a Chinese dataset: the cluster name it returns (plus the deterministic
+// hash suffix) is what the nav tree displays, and its "summary" is the cluster
+// description the artifacts page shows.
+func TestNavPromptsConstrainLanguage(t *testing.T) {
+ naming := navNamingPrompt("赵云的英勇事迹与忠诚品质\n周瑜是东吴大都督……")
+ if !strings.Contains(naming, "same language as the excerpts") {
+ t.Errorf("naming prompt lost its language constraint: %s", naming)
+ }
+ // The source text and the JSON contract must survive the extra instruction.
+ if !strings.Contains(naming, "赵云的英勇事迹与忠诚品质") {
+ t.Error("naming prompt dropped the source text")
+ }
+ if !strings.Contains(naming, `Return ONLY JSON: {"name": "<2-6 word topic title>"`) {
+ t.Error("naming prompt dropped the JSON contract")
+ }
+
+ merging := navMergePrompt("Three Kingdoms Historical Figures", "赵云的英勇事迹与忠诚品质")
+ if !strings.Contains(merging, "same language as the New description") {
+ t.Errorf("merge prompt lost its language constraint: %s", merging)
+ }
+ if !strings.Contains(merging, "Existing: Three Kingdoms Historical Figures") ||
+ !strings.Contains(merging, "New: 赵云的英勇事迹与忠诚品质") {
+ t.Error("merge prompt dropped one of the two descriptions")
+ }
+ if !strings.Contains(merging, "Return ONLY the merged text, no commentary.") {
+ t.Error("merge prompt dropped its output contract")
+ }
+}
+
+// TestNavSummaryFromReply pins the naming reply contract (Python
+// _llm_create_summary): "name" is the topic title, "summary" falls back to
+// "result" and then to the source text. An empty name is surfaced as-is so the
+// caller keeps its deterministic "<summary first line> <hash>" name.
+func TestNavSummaryFromReply(t *testing.T) {
+ tests := []struct {
+ name string
+ reply map[string]any
+ fallback string
+ wantName string
+ wantSummary string
+ }{
+ {
+ name: "name and summary",
+ reply: map[string]any{"name": "三国名将事迹", "summary": "三国名将的事迹概览。"},
+ fallback: "source summary",
+ wantName: "三国名将事迹",
+ wantSummary: "三国名将的事迹概览。",
+ },
+ {
+ name: "summary key missing falls back to result",
+ reply: map[string]any{"name": " topic ", "result": "merged text"},
+ fallback: "source summary",
+ wantName: "topic",
+ wantSummary: "merged text",
+ },
+ {
+ name: "no usable fields keeps the source text",
+ reply: map[string]any{},
+ fallback: "source summary",
+ wantName: "",
+ wantSummary: "source summary",
+ },
+ {
+ name: "name missing leaves the caller to fall back",
+ reply: map[string]any{"summary": "only a summary"},
+ fallback: "source summary",
+ wantName: "",
+ wantSummary: "only a summary",
+ },
+ {
+ name: "non-string values are ignored, not stringified",
+ reply: map[string]any{"name": 42, "summary": map[string]any{"x": 1}},
+ fallback: "source summary",
+ wantName: "",
+ wantSummary: "source summary",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ gotName, gotSummary := navSummaryFromReply(tt.reply, tt.fallback)
+ if gotName != tt.wantName || gotSummary != tt.wantSummary {
+ t.Errorf("navSummaryFromReply() = (%q, %q), want (%q, %q)",
+ gotName, gotSummary, tt.wantName, tt.wantSummary)
+ }
+ })
+ }
+}
+
+// TestNavMergedFromReply pins the merge reply contract (Python _llm_merge), with
+// the deliberate deviation documented on the function: a plain-text reply is
+// used (the prompt asks for "only the merged text"), while a JSON reply prefers
+// "merged" then "result" and an unusable reply keeps the existing description.
+func TestNavMergedFromReply(t *testing.T) {
+ tests := []struct {
+ name string
+ reply string
+ existing string
+ want string
+ }{
+ {
+ name: "plain text reply is the merged summary",
+ reply: "三国名将与曹操的对比,两人才能各具特色。",
+ existing: "old description",
+ want: "三国名将与曹操的对比,两人才能各具特色。",
+ },
+ {
+ name: "merged key wins",
+ reply: `{"merged": "fused description"}`,
+ existing: "old description",
+ want: "fused description",
+ },
+ {
+ name: "result key is accepted",
+ reply: `{"result": "fused via result"}`,
+ existing: "old description",
+ want: "fused via result",
+ },
+ {
+ name: "json without the expected keys keeps the existing text",
+ reply: `{"unexpected": "value"}`,
+ existing: "old description",
+ want: "old description",
+ },
+ {
+ name: "empty reply keeps the existing text",
+ reply: "",
+ existing: "old description",
+ want: "old description",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := navMergedFromReply(tt.reply, tt.existing); got != tt.want {
+ t.Errorf("navMergedFromReply(%q) = %q, want %q", tt.reply, got, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.go
index 49190bc78c..bfe0ad8c53 100644
--- a/internal/service/nlp/datasetnav.go
+++ b/internal/service/nlp/datasetnav.go
@@ -23,10 +23,13 @@ import (
"sort"
"strings"
+ "ragflow/internal/common"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
"ragflow/internal/service/nav"
"ragflow/internal/tokenizer"
+
+ "go.uber.org/zap"
)
// Dataset-nav constants. These mirror Python's dataset_nav.py: nav rows live in
@@ -47,8 +50,22 @@ const (
// the children-search cost when the cluster is plausibly overfull.
navMaxFanout = 64 // max direct children before rebalance
navMaxDocsPerCluster = 50 // max docs per leaf cluster before split
+
+ // navTreeReadLimit bounds the single read a search uses to resolve the
+ // ancestor path of its hits. A nav tree is small by construction (fanout,
+ // docs-per-cluster and depth are all capped), so this is a guard against a
+ // pathological dataset rather than a paging budget.
+ navTreeReadLimit = 10000
)
+// navNodeSelectFields are the columns every nav tree read needs: the label
+// (title_kwd), the Python-era name column, the parent edge, the payload
+// (description), and the leaf's document id.
+var navNodeSelectFields = []string{
+ "name", "title_kwd", "parent_kwd", "depth_int",
+ "content_with_weight", "doc_count_int", "type_kwd", "doc_id",
+}
+
// NavEmbedder embeds text. tenantID lets a production implementation resolve
// the tenant's embedding model. Kept as an interface so tests inject a stub.
type NavEmbedder interface {
@@ -86,10 +103,10 @@ func NewNavService(embed NavEmbedder) *NavService {
return &NavService{embed: embed}
}
-// SetNavMergeLLM installs the optional LLM used for cluster-description merging
-// and summary naming (mirroring Python dataset_nav._llm_merge/_llm_create_summary).
-// When nil (the default), NavService falls back to deterministic naming and
-// concatenated descriptions.
+// SetNavMergeLLM installs the optional LLM used for cluster naming and
+// description merging (mirroring Python dataset_nav._llm_create_summary /
+// _llm_merge). When nil (the default), clusters keep the deterministic
+// "<summary first line> <hash>" name and their existing description.
func (s *NavService) SetNavMergeLLM(llm nav.NavMergeLLM) {
s.llm = llm
}
@@ -149,24 +166,27 @@ func (s *NavService) navSearch(ctx context.Context, tenantID, kbID string, filte
return res.Chunks, res.Total, nil
}
-// ListClusters returns the depth-0 clusters (parent_kwd=root). When keywords
-// is non-empty, it searches all navigation rows by their indexed summary and
-// returns matching nodes, mirroring the Python endpoint's search mode.
+// ListClusters returns the depth-0 clusters (parent_kwd=root). When keywords is
+// non-empty it searches the dataset's documents and returns the pruned forest
+// around the hits (see searchNavForest) instead: one tree per root cluster,
+// holding only the cluster path down to each matched nav_doc leaf.
+//
+// The total is a cluster count in both modes: the depth-0 clusters when browsing,
+// the clusters holding a match when searching.
func (s *NavService) ListClusters(ctx context.Context, tenantID, kbID, keywords string, page, pageSize int) ([]nav.NavNode, int64, error) {
if pageSize <= 0 {
pageSize = 100
}
+ if strings.TrimSpace(keywords) != "" {
+ return s.searchNavForest(ctx, tenantID, kbID, keywords, pageSize)
+ }
offset := page * pageSize
filter := navFilter(map[string]interface{}{
"type_kwd": []string{"nav_cluster"},
"parent_kwd": []string{navRootParent},
})
- if strings.TrimSpace(keywords) != "" {
- filter = navFilter(nil)
- }
chunks, total, err := s.navSearch(ctx, tenantID, kbID, filter,
- []string{"name", "title_kwd", "content_with_weight", "doc_count_int", "type_kwd", "doc_id"}, offset, pageSize,
- navKeywordExpressions(keywords))
+ navNodeSelectFields, offset, pageSize, navKeywordExpressions(keywords))
if err != nil {
return nil, 0, err
}
@@ -186,7 +206,7 @@ func (s *NavService) ListChildren(ctx context.Context, tenantID, kbID, name, key
offset := page * pageSize
chunks, total, err := s.navSearch(ctx, tenantID, kbID,
navFilter(map[string]interface{}{"parent_kwd": []string{name}}),
- []string{"name", "title_kwd", "content_with_weight", "doc_count_int", "type_kwd", "doc_id"}, offset, pageSize,
+ navNodeSelectFields, offset, pageSize,
navKeywordExpressions(keywords))
if err != nil {
return nil, 0, err
@@ -203,6 +223,135 @@ func (s *NavService) ListChildren(ctx context.Context, tenantID, kbID, name, key
return nodes, total, nil
}
+// searchNavForest resolves a keyword search into the pruned forest the nav tree
+// renders: the matched nav_doc leaves PLUS the cluster path from each match up to
+// its root cluster, so every root cluster stays ONE tree holding only the
+// branches that lead to a match. The rows carry parent_kwd (the tree edge) and
+// matched (hits only), so the caller nests the flat list without another request.
+//
+// The total counts the clusters the hits landed in (see searchNavClusterCount):
+// the same quantity the header shows while browsing. Matching is document-level
+// on purpose (Python's tree search routes to documents too): a cluster appears
+// only as the path above a matched leaf, so a cluster's own summary text alone
+// does not surface a tree.
+func (s *NavService) searchNavForest(ctx context.Context, tenantID, kbID, keywords string, limit int) ([]nav.NavNode, int64, error) {
+ matches, _, err := s.navSearch(ctx, tenantID, kbID,
+ navFilter(map[string]interface{}{"type_kwd": []string{nav.TypeNavDoc}}),
+ navNodeSelectFields, 0, limit, navKeywordExpressions(keywords))
+ if err != nil {
+ return nil, 0, err
+ }
+ total := searchNavClusterCount(matches)
+ if len(matches) == 0 {
+ return []nav.NavNode{}, total, nil
+ }
+ // One read of the dataset's nav rows resolves every ancestor chain at once.
+ // A nav tree is small by construction (fanout, docs-per-cluster and depth are
+ // all capped), so this beats a parent-by-parent walk of navMaxDepth queries.
+ rows, _, err := s.navSearch(ctx, tenantID, kbID, navFilter(nil), navNodeSelectFields, 0, navTreeReadLimit, nil)
+ if err != nil {
+ // The path lookup failed; the hits are still a valid — if flat — answer.
+ common.Warn("datasetnav: nav path lookup for the search forest failed",
+ zap.String("kb_id", kbID), zap.Error(err))
+ return s.navNodesFromRows(matches), total, nil
+ }
+ clusters := make(map[string]map[string]interface{}, len(rows))
+ for _, r := range rows {
+ if firstStringValue(r["type_kwd"]) != nav.TypeNavCluster {
+ continue
+ }
+ if key := navClusterRowKey(r); key != "" {
+ clusters[key] = r
+ }
+ }
+
+ kept := make(map[string]map[string]interface{}, len(matches))
+ hit := make(map[string]bool, len(matches))
+ for _, r := range matches {
+ docID := firstStringValue(r["doc_id"])
+ if docID == "" {
+ continue
+ }
+ key := navDocRowKey(docID)
+ kept[key] = r
+ hit[key] = true
+ // Walk up to (but excluding) the root sentinel. navMaxDepth bounds the
+ // walk so a corrupt parent cycle can never spin.
+ parent := firstStringValue(r["parent_kwd"])
+ for depth := 0; parent != "" && parent != navRootParent && depth < navMaxDepth; depth++ {
+ cluster, ok := clusters[parent]
+ if !ok {
+ break // orphaned edge: the path above it is unknown
+ }
+ kept[navClusterKey(parent)] = cluster
+ parent = firstStringValue(cluster["parent_kwd"])
+ }
+ }
+ // Deterministic order (shallowest first, then by key): the caller re-nests
+ // anyway, but a stable order keeps the response diffable and testable.
+ keys := make([]string, 0, len(kept))
+ for key := range kept {
+ keys = append(keys, key)
+ }
+ sort.Slice(keys, func(i, j int) bool {
+ if di, dj := intValue(kept[keys[i]]["depth_int"]), intValue(kept[keys[j]]["depth_int"]); di != dj {
+ return di < dj
+ }
+ return keys[i] < keys[j]
+ })
+ nodes := make([]nav.NavNode, 0, len(keys))
+ for _, key := range keys {
+ node := s.nodeFromRow(kept[key], "doc")
+ node.Matched = hit[key]
+ nodes = append(nodes, node)
+ }
+ return nodes, total, nil
+}
+
+// searchNavClusterCount returns the number of clusters a search's hits landed in
+// — the header's number in search mode, like the depth-0 cluster count it shows
+// while browsing. A hit leaf sits in exactly one cluster (its parent_kwd), so
+// distinct parents count the clusters touched: two hits in one cluster count
+// once, and an ancestor shared by hits in different sub-clusters is a path row,
+// not a second cluster.
+func searchNavClusterCount(matches []map[string]interface{}) int64 {
+ seen := make(map[string]bool, len(matches))
+ for _, r := range matches {
+ // A leaf with no cluster edge (no writer produces one) has no cluster to
+ // count.
+ if key := firstStringValue(r["parent_kwd"]); key != "" && key != navRootParent {
+ seen[key] = true
+ }
+ }
+ return int64(len(seen))
+}
+
+// navNodesFromRows maps engine rows to NavNodes, all marked as hits (the degrade
+// path of searchNavForest, where no path rows were resolved).
+func (s *NavService) navNodesFromRows(rows []map[string]interface{}) []nav.NavNode {
+ nodes := make([]nav.NavNode, 0, len(rows))
+ for _, r := range rows {
+ node := s.nodeFromRow(r, "doc")
+ node.Matched = true
+ nodes = append(nodes, node)
+ }
+ return nodes
+}
+
+// navClusterRowKey is the key a cluster's children reference through parent_kwd:
+// Python's writer stores it in `name`, Go's in `title_kwd`.
+func navClusterRowKey(row map[string]interface{}) string {
+ if name := firstStringValue(row["name"]); name != "" {
+ return name
+ }
+ return firstStringValue(row["title_kwd"])
+}
+
+// navDocRowKey / navClusterKey namespace the two row identities so a cluster and
+// a document can never collide in the search forest's keyed sets.
+func navDocRowKey(docID string) string { return "doc:" + docID }
+func navClusterKey(name string) string { return "cluster:" + name }
+
// navKeywordExpressions builds the lexical search expression used by the REST
// navigation endpoints. These are the same indexed summary fields used by the
// Python navigation search, with a stronger weight on the main content field.
@@ -222,10 +371,13 @@ func navKeywordExpressions(keywords string) []interface{} {
}
// nodeFromRow converts an engine row into a NavNode.
+//
+// The label is title_kwd — the summary's first line: one column serves both the
+// tree and the model-facing readers (NavHit.Name), so no separate display column
+// is written. An explicit readable `name` column (Python's writer stores one)
+// wins when present.
func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string) nav.NavNode {
name := firstStringValue(row["title_kwd"])
- // Prefer an explicit readable "name" column (Python rows carry one); fall
- // back to title_kwd.
if n := firstStringValue(row["name"]); n != "" {
name = n
}
@@ -243,6 +395,7 @@ func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string
DocCount: intValue(row["doc_count_int"]),
Type: fallbackType,
DocID: firstStringValue(row["doc_id"]),
+ Parent: firstStringValue(row["parent_kwd"]),
}
if t := firstStringValue(row["type_kwd"]); t != "" {
if t == "nav_cluster" {
@@ -476,11 +629,15 @@ func navIntersectScope(docIDs []string, allowed map[string]struct{}) []string {
return out
}
-// SummariesByDocIDs returns the nav_doc summary keyed by doc_id for the given
-// documents. It mirrors Python dataset_api_service._nav_doc_summaries: it reads
-// the nav_doc rows (compile_kwd=dataset_nav, type_kwd=nav_doc) for the doc_ids
-// and yields, per doc, the readable nav name when present, else the payload
-// description. Documents with no nav_doc row are absent from the result.
+// SummariesByDocIDs returns the nav_doc description — the document's overall
+// summary — keyed by doc_id for the given documents. It mirrors Python
+// dataset_api_service._nav_doc_summaries: it reads the nav_doc rows
+// (compile_kwd=dataset_nav, type_kwd=nav_doc) for the doc_ids and yields, per
+// doc, the payload description. The description wins over the row's label on
+// purpose: callers render this value to the model as the routed document's
+// <summary>, and title_kwd holds only the summary's first line (length-capped).
+// A row whose payload carries no description falls back to its readable label; a
+// document with no nav_doc row is absent from the result.
func (s *NavService) SummariesByDocIDs(ctx context.Context, tenantID, kbID string, docIDs []string) map[string]string {
if len(docIDs) == 0 {
return map[string]string{}
@@ -501,24 +658,25 @@ func (s *NavService) SummariesByDocIDs(ctx context.Context, tenantID, kbID strin
if docID == "" {
continue
}
- // Prefer an explicit readable name, else the payload description, matching
- // nodeFromRow's label rules (a raw 32-hex id is not a human label).
- name := firstStringValue(c["name"])
- if name == "" {
- name = firstStringValue(c["title_kwd"])
- }
- if graphIsRawID(name) {
- name = ""
+ summary := ""
+ if payload, ok := c["content_with_weight"].(string); ok {
+ var m map[string]interface{}
+ if json.Unmarshal([]byte(payload), &m) == nil {
+ if d, ok := m["description"].(string); ok {
+ summary = strings.TrimSpace(d)
+ }
+ }
}
- summary := name
if summary == "" {
- if payload, ok := c["content_with_weight"].(string); ok {
- var m map[string]interface{}
- if json.Unmarshal([]byte(payload), &m) == nil {
- if d, ok := m["description"].(string); ok {
- summary = strings.TrimSpace(d)
- }
- }
+ // No description in the payload: fall back to the stored label
+ // (readable name, else title_kwd), rejecting raw ids — a 32-hex id or
+ // "cluster_<hash>" is not a human label.
+ summary = firstStringValue(c["name"])
+ if summary == "" {
+ summary = firstStringValue(c["title_kwd"])
+ }
+ if graphIsRawID(summary) {
+ summary = ""
}
}
if summary != "" {
@@ -568,18 +726,35 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"type_kwd": []string{nav.TypeNavDoc},
"doc_id": []string{in.DocID},
}),
- []string{"content_with_weight", "content_ltks", "content_sm_ltks"}, 0, 1, nil)
+ []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd"}, 0, 1, nil)
if err != nil {
return err
}
if len(existing) > 0 {
+ // The stored label can drift without the summary changing, so a recompile
+ // refreshes it in place instead of leaving a stale row: a row written by an
+ // older label rule (title_kwd holding the file name) goes back to the
+ // summary's first line, or the model-facing readers would keep seeing a
+ // file name. The tokenized text is (re)built when it is missing.
+ title := navDocTitle(in.Summary)
+ titleStale := firstStringValue(existing[0]["title_kwd"]) != title
+ ltksStale := firstStringValue(existing[0]["content_ltks"]) == "" ||
+ firstStringValue(existing[0]["content_sm_ltks"]) == ""
if payload, ok := existing[0]["content_with_weight"].(string); ok {
var m map[string]interface{}
if err := json.Unmarshal([]byte(payload), &m); err == nil {
if d, _ := m["description"].(string); d == in.Summary {
- if firstStringValue(existing[0]["content_ltks"]) != "" && firstStringValue(existing[0]["content_sm_ltks"]) != "" {
+ if !titleStale && !ltksStale {
return nil // unchanged
}
+ updates := map[string]interface{}{}
+ if titleStale {
+ updates["title_kwd"] = title
+ }
+ if ltksStale {
+ updates["content_ltks"] = contentLtks
+ updates["content_sm_ltks"] = contentSmLtks
+ }
return de.UpdateChunks(ctx,
map[string]interface{}{
"compile_kwd": []string{navCompileKwd},
@@ -587,10 +762,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"doc_id": []string{in.DocID},
"kb_id": in.KbID,
},
- map[string]interface{}{
- "content_ltks": contentLtks,
- "content_sm_ltks": contentSmLtks,
- }, s.navIndexName(in.TenantID), in.KbID)
+ updates, s.navIndexName(in.TenantID), in.KbID)
}
}
}
@@ -619,6 +791,14 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
if bestName != "" && sim >= navMergeThreshold {
parent := bestName
+ // Fuse this document's summary into the cluster description and re-embed
+ // the cluster with the merged text (Python upsert_dataset_nav_doc's merge
+ // branch), so the cluster keeps describing every document it holds and the
+ // next placement scores against the merged topic instead of the first
+ // document's summary alone.
+ if err := s.mergeClusterDescription(ctx, de, in.TenantID, in.KbID, bestName, in.Summary); err != nil {
+ return err
+ }
if err := s.appendDocToCluster(ctx, de, in.TenantID, in.KbID, bestName, in.DocID,
fmt.Sprintf("q_%d_vec", len(vec))); err != nil {
return err
@@ -632,10 +812,12 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"compile_kwd": navCompileKwd,
"available_int": 0,
"type_kwd": "nav_doc",
- // The nav_doc's display name is a readable title derived from the summary
- // (Python _clean_title/_fallback_title) — NOT the raw doc id, which is
- // meaningless in the UI. The doc_id is still stored for lookups.
- "title_kwd": cleanTitle(fallbackTitle(in.Summary)),
+ // The nav_doc's label is a readable title derived from the summary
+ // (Python _clean_title/_fallback_title) — never the raw doc id,
+ // which is meaningless in the UI. The doc_id is still stored for
+ // lookups; NavHit.Name reads this same column, so the tree and the
+ // model-facing readers stay on one label.
+ "title_kwd": navDocTitle(in.Summary),
"parent_kwd": parent,
// The nav_doc sits one level below its (possibly nested) parent
// cluster, so its depth is parentDepth+1 — not a hard-coded 1.
@@ -651,15 +833,22 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
return err
}
- // A similar-but-not-mergeable cluster creates a sibling sub-cluster (Python
- // _MIN_SIM=0.50); otherwise a fresh root cluster. This keeps the nav tree
- // from degrading into one root per document.
+ // A similar-but-not-mergeable document opens its own cluster (Python
+ // _MIN_SIM=0.50). Where it lands follows the best match's depth:
+ //
+ // - a ROOT-level best match opens a ROOT-level cluster, beside it. That is
+ // the case a fresh dataset always reaches, and it is what Python's branch
+ // does: it attaches the new cluster to `best_parent`, which for a root
+ // best match is the root sentinel. Hiding the new topic one level down
+ // instead (what this branch used to do) meant a second topic never showed
+ // up in the tree's top level.
+ // - a best match that is itself nested keeps the child semantics: the new
+ // cluster sits one level deeper, which is what keeps a large tree from
+ // flattening into one root per topic.
parent := navRootParent
depth := 0
- if bestName != "" && sim >= navMinSim {
+ if bestName != "" && sim >= navMinSim && bestDepth > 0 {
parent = bestName
- // A sibling of the (possibly nested) best cluster is one level deeper
- // than it, so depth = parentDepth+1 — not a hard-coded 1.
depth = bestDepth + 1
}
// New cluster: when an LLM is configured, generate a short name + summary
@@ -707,7 +896,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"compile_kwd": navCompileKwd,
"available_int": 0,
"type_kwd": "nav_doc",
- "title_kwd": cleanTitle(fallbackTitle(in.Summary)),
+ "title_kwd": navDocTitle(in.Summary),
"parent_kwd": name,
"depth_int": depth + 1,
"doc_id": in.DocID,
@@ -788,9 +977,10 @@ func (s *NavService) cleanupEmptyCluster(ctx context.Context, tenantID, kbID, cl
// findBestCluster finds the best-matching cluster via level-by-level descent
// (mirroring Python _find_best_cluster). It KNNs the current level's clusters
-// and, when the best match is >= recurse threshold, descends into that cluster's
-// children. Returns the best cluster name, similarity, and its depth (0 = root)
-// so callers can assign consistent child depth_int values.
+// and, while the match is >= recurse threshold, descends into that cluster's
+// children. Returns the DEEPEST cluster reached — the strongest match is not
+// kept, exactly like Python — plus its similarity and depth (0 = root), so the
+// caller can assign consistent child depth_int values.
func (s *NavService) findBestCluster(ctx context.Context, tenantID, kbID string, vec []float32) (string, float64, int, error) {
f64 := f32ToF64Slice(vec)
parent := navRootParent
@@ -821,10 +1011,14 @@ func (s *NavService) findBestCluster(ctx context.Context, tenantID, kbID string,
}
name := firstStringValue(chunks[0]["title_kwd"])
sim := rowScore(chunks[0])
- // Keep the STRONGEST match seen so far across all levels, so a strong
- // ancestor is never displaced by a weaker descendant. Record its depth
- // so the caller can set consistent child depth_int values.
- if sim > bestSim {
+ // Adopt-and-overwrite, mirroring Python _find_best_cluster: the result is
+ // the DEEPEST cluster reached while the descent keeps clearing
+ // navRecurse, not the strongest one seen. A strong ancestor therefore
+ // does NOT shadow a weaker-but-recurseable descendant — the placement
+ // then decides on the descendant's own similarity. The root (level 0) is
+ // adopted unconditionally, exactly as Python seeds its `best` with the
+ // top-1 root; a child is adopted only when it still clears the threshold.
+ if level == 0 || sim >= navRecurse {
bestName, bestSim, bestDepth = name, sim, level
}
// Descend only while the current match is strong enough that a deeper
@@ -1105,16 +1299,89 @@ func clusterDepth(clusterChunks []map[string]interface{}) int {
return intValue(clusterChunks[0]["depth_int"])
}
-// llmMergeDescription merges a set of source descriptions into one, using the
-// optional LLM when available (temperature 0.1, mirroring Python _llm_merge);
-// without an LLM it concatenates them deterministically.
+// mergeClusterDescription fuses one document's summary into an existing
+// cluster's description and re-embeds the cluster with the merged text
+// (mirroring Python upsert_dataset_nav_doc's merge branch: _llm_merge, then a
+// re-embed of the changed description). Nothing is written when there is no LLM
+// — Python's _llm_merge returns the existing text when chat_mdl is nil — or when
+// the model reply leaves the description unchanged, so a no-LLM deployment keeps
+// its previous behaviour.
+func (s *NavService) mergeClusterDescription(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docSummary string) error {
+ docSummary = strings.TrimSpace(docSummary)
+ if docSummary == "" {
+ return nil
+ }
+ rows, _, err := s.navSearch(ctx, tenantID, kbID,
+ navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),
+ []string{"content_with_weight"}, 0, 1, nil)
+ if err != nil {
+ return err
+ }
+ if len(rows) == 0 {
+ return nil
+ }
+ payload := map[string]interface{}{}
+ if raw, ok := rows[0]["content_with_weight"].(string); ok {
+ _ = json.Unmarshal([]byte(raw), &payload)
+ }
+ oldDesc := firstStringValue(payload["description"])
+ if oldDesc == "" {
+ // Nothing to fuse with: the row carries no description to preserve.
+ return nil
+ }
+ merged := strings.TrimSpace(s.llmMergeDescription(ctx, tenantID, []string{oldDesc, docSummary}))
+ if merged == "" || merged == oldDesc {
+ return nil
+ }
+ // Mutate the parsed payload so any extra keys a writer stored there (Python
+ // carries keywords/entities/graph_content) survive the rewrite.
+ payload["description"] = merged
+ updates := map[string]interface{}{"content_with_weight": payloadJSONNav(payload)}
+ // The description is part of the cluster's keyword index, so its tokens are
+ // refreshed alongside it.
+ if ltks, err := tokenizer.Tokenize(merged); err == nil && ltks != "" {
+ updates["content_ltks"] = ltks
+ if smLtks, err := tokenizer.FineGrainedTokenize(ltks); err == nil && smLtks != "" {
+ updates["content_sm_ltks"] = smLtks
+ }
+ }
+ if s.embed != nil {
+ if vecs, err := s.embed.Encode(ctx, tenantID, []string{merged}); err == nil && len(vecs) > 0 && len(vecs[0]) > 0 {
+ updates[fmt.Sprintf("q_%d_vec", len(vecs[0]))] = f32ToF64Slice(vecs[0])
+ }
+ }
+ return de.UpdateChunks(ctx,
+ map[string]interface{}{
+ "compile_kwd": []string{navCompileKwd},
+ "type_kwd": []string{"nav_cluster"},
+ "title_kwd": []string{clusterName},
+ "kb_id": kbID,
+ }, updates, s.navIndexName(tenantID), kbID)
+}
+
+// llmMergeDescription merges a set of source descriptions into one via the
+// optional LLM (temperature 0.1, mirroring Python _llm_merge). texts is
+// [existing description, new document summary]. Without an LLM — or when the
+// model reply is unusable — the existing description is returned unchanged,
+// mirroring Python's fallback (it keeps the old summary rather than
+// concatenating).
func (s *NavService) llmMergeDescription(ctx context.Context, tenantID string, texts []string) string {
+ existing := ""
+ if len(texts) > 0 {
+ existing = texts[0]
+ }
if s.llm != nil && len(texts) > 0 {
- if merged, err := s.llm.Merge(ctx, tenantID, texts); err == nil && strings.TrimSpace(merged) != "" {
+ merged, err := s.llm.Merge(ctx, tenantID, texts)
+ if err != nil {
+ // Keep the stored description; the warning is the only signal that the
+ // configured chat model could not be used (Python logs the same way).
+ common.Warn("datasetnav: LLM cluster-description merge failed, keeping the existing description",
+ zap.String("tenant_id", tenantID), zap.Error(err))
+ } else if strings.TrimSpace(merged) != "" {
return merged
}
}
- return strings.Join(texts, "\n")
+ return existing
}
// cleanTitle normalizes a raw title/summary into a one-line, length-capped
@@ -1142,6 +1409,16 @@ func fallbackTitle(summary string) string {
return line
}
+// navDocTitle is the nav_doc leaf's label, stored in title_kwd: the summary's
+// first line (Python _clean_title/_fallback_title). One column serves both
+// readers — nodeFromRow renders it in the tree, and NavHit.Name carries it to
+// the model as the routed document's <summary> (tool_navigation.go) — so it must
+// stay a CONTENT label; a file name here would tell the model nothing about the
+// document.
+func navDocTitle(summary string) string {
+ return cleanTitle(fallbackTitle(summary))
+}
+
// readableClusterName returns a readable yet unique nav-cluster key of the form
// "<title> <8-hex>" (mirroring Python dataset_nav._readable_cluster_name): the
// title keeps the node name human-readable, the short hash of the seed keeps the
@@ -1154,14 +1431,24 @@ func readableClusterName(title, seed string) string {
return t + " " + shortHash8(seed)
}
-// llmCreateSummary builds a short cluster name + summary for a source text via
-// the optional LLM (mirroring Python _llm_create_summary); without an LLM it
-// falls back to a readable name derived from the summary's first words plus a
-// short hash, so nav clusters are human-readable instead of "cluster_<hash>".
+// llmCreateSummary builds a cluster's name + summary for a source text. The
+// name is always run through readableClusterName, so it keeps the " <8-hex>"
+// suffix that makes it unique per KB and addressable as the cluster key
+// (parent_kwd / row id) — Python does the same in its callers
+// (_readable_cluster_name(new_title, summary)). With an LLM the title part is
+// the model's topic name; without one it is the summary's first line, so nav
+// clusters stay human-readable instead of "cluster_<hash>".
func (s *NavService) llmCreateSummary(ctx context.Context, tenantID, text string) (name, summary string) {
if s.llm != nil {
- if n, sm, err := s.llm.CreateSummary(ctx, tenantID, text); err == nil && n != "" {
- return n, sm
+ n, sm, err := s.llm.CreateSummary(ctx, tenantID, text)
+ switch {
+ case err != nil:
+ // Fall back to the deterministic name; the warning is the only signal
+ // that the configured chat model could not be used.
+ common.Warn("datasetnav: LLM cluster naming failed, using the deterministic name",
+ zap.String("tenant_id", tenantID), zap.Error(err))
+ case n != "":
+ return readableClusterName(n, text), sm
}
}
return readableClusterName(fallbackTitle(text), text), text
diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.go
index 753aad9d64..5a70733828 100644
--- a/internal/service/nlp/datasetnav_test.go
+++ b/internal/service/nlp/datasetnav_test.go
@@ -69,10 +69,15 @@ func (m *memNavEngine) DeleteChunks(_ context.Context, cond map[string]interface
func (m *memNavEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
var matched []map[string]interface{}
- hasDense := len(req.MatchExprs) > 0
+ // This engine implements the DENSE leg only. A lexical match expression is
+ // deliberately a no-op rather than a dense one: treating a text expression
+ // as dense would look for a q_0_vec column and silently drop every row, so a
+ // keyword search would look broken in tests.
+ hasDense := false
var queryVec []float64
- if hasDense {
+ if len(req.MatchExprs) > 0 {
if de, ok := req.MatchExprs[0].(*types.MatchDenseExpr); ok {
+ hasDense = true
queryVec = de.EmbeddingData
}
}
@@ -365,6 +370,210 @@ func TestNavNode_JSONShape_SnakeCase(t *testing.T) {
t.Errorf("NavNode JSON leaked PascalCase key %q; got %s", bad, b)
}
}
+ // A search hit carries the tree edge and the hit flag. Both are optional:
+ // the ordinary tree reads (root clusters, children) never set them, so they
+ // must stay out of the payload there.
+ hit := nav.NavNode{Name: "doc.pdf", Type: "doc", Parent: "topic 8738e200", Matched: true}
+ hb, err := json.Marshal(hit)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var hm map[string]interface{}
+ if err := json.Unmarshal(hb, &hm); err != nil {
+ t.Fatal(err)
+ }
+ if hm["parent_kwd"] != "topic 8738e200" || hm["matched"] != true {
+ t.Errorf("search hit JSON = %s, want parent_kwd + matched", hb)
+ }
+ if _, ok := m["matched"]; ok {
+ t.Errorf("a non-hit NavNode must not carry matched; got %s", b)
+ }
+}
+
+// TestNavService_ListClusters_SearchReturnsPrunedForest pins the search shape:
+// the hits come back TOGETHER with the cluster path above them, so the caller
+// nests the flat list into one tree per root cluster instead of rendering every
+// hit — and every matched subtree — as its own tree.
+func TestNavService_ListClusters_SearchReturnsPrunedForest(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ const (
+ rootCluster = "三国 11111111"
+ subCluster = "名将 22222222"
+ )
+ rows := []map[string]interface{}{
+ {
+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavCluster,
+ "title_kwd": rootCluster, "parent_kwd": navRootParent, "depth_int": 0,
+ "doc_count_int": 2, "doc_ids_kwd": []string{"d1", "d2"},
+ "content_with_weight": `{"type":"nav_cluster","description":"三国主题"}`,
+ },
+ {
+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavCluster,
+ "title_kwd": subCluster, "parent_kwd": rootCluster, "depth_int": 1,
+ "doc_count_int": 1, "doc_ids_kwd": []string{"d1"},
+ "content_with_weight": `{"type":"nav_cluster","description":"名将主题"}`,
+ },
+ {
+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,
+ "doc_id": "d1", "parent_kwd": subCluster, "depth_int": 2,
+ "title_kwd": "三国名将事迹概要", "doc_count_int": 1,
+ "content_with_weight": `{"type":"nav_doc","description":"三国名将事迹概要\n正文"}`,
+ },
+ // A second leaf in subCluster: two hits in ONE cluster must count once.
+ {
+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,
+ "doc_id": "d4", "parent_kwd": subCluster, "depth_int": 2,
+ "title_kwd": "名将功过评析", "doc_count_int": 1,
+ "content_with_weight": `{"type":"nav_doc","description":"名将功过评析"}`,
+ },
+ {
+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,
+ "doc_id": "d2", "parent_kwd": rootCluster, "depth_int": 1,
+ "title_kwd": "其他概要", "doc_count_int": 1,
+ "content_with_weight": `{"type":"nav_doc","description":"其他概要"}`,
+ },
+ // An orphan whose parent cluster is gone: the hit is still returned (as
+ // its own root) rather than dropped.
+ {
+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,
+ "doc_id": "d3", "parent_kwd": "missing-cluster", "depth_int": 1,
+ "title_kwd": "孤儿概要", "doc_count_int": 1,
+ "content_with_weight": `{"type":"nav_doc","description":"孤儿概要"}`,
+ },
+ }
+ if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1"); err != nil {
+ t.Fatal(err)
+ }
+
+ nodes, total, err := ns.ListClusters(t.Context(), "t1", "kb1", "任意关键词", 0, 100)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // The test engine ignores the lexical expression, so all four nav_doc rows
+ // are hits. total is the CLUSTER count — the same quantity the nav tree
+ // header shows while browsing — not the hit count: d1 and d4 share
+ // subCluster (one cluster, two hits), d2 sits directly in rootCluster, and
+ // d3's cluster edge dangles. The path rows above the hits are not counted.
+ if total != 3 {
+ t.Errorf("search total = %d, want 3 (the clusters the hits landed in, not the 4 hits)", total)
+ }
+ byID := make(map[string]nav.NavNode, len(nodes))
+ for _, n := range nodes {
+ if n.Type == "doc" {
+ byID[n.DocID] = n
+ }
+ }
+ if len(byID) != 4 {
+ t.Fatalf("search forest returned %d doc nodes, want 4: %+v", len(byID), nodes)
+ }
+ // The cluster path above d1 (sub-cluster then root) must be present so the
+ // tree stays ONE tree, and it must not be flagged as a hit.
+ var gotRoot, gotSub bool
+ for _, n := range nodes {
+ switch n.Name {
+ case rootCluster:
+ gotRoot = true
+ if n.Matched {
+ t.Errorf("path cluster %q marked as a hit", n.Name)
+ }
+ if n.Parent != navRootParent {
+ t.Errorf("root cluster parent = %q, want %q", n.Parent, navRootParent)
+ }
+ case subCluster:
+ gotSub = true
+ if n.Matched {
+ t.Errorf("path cluster %q marked as a hit", n.Name)
+ }
+ if n.Parent != rootCluster {
+ t.Errorf("sub cluster parent = %q, want %q", n.Parent, rootCluster)
+ }
+ }
+ }
+ if !gotRoot || !gotSub {
+ t.Fatalf("search forest is missing the path above d1 (root=%v sub=%v): %+v", gotRoot, gotSub, nodes)
+ }
+ // The leaf keeps both the tree edge and its label (title_kwd, the summary's
+ // first line — the same column the model-facing readers use).
+ if got := byID["d1"]; got.Parent != subCluster || !got.Matched || got.Name != "三国名将事迹概要" {
+ t.Errorf("hit d1 = %+v, want parent=%q matched=true name=三国名将事迹概要", got, subCluster)
+ }
+ if got := byID["d4"]; got.Parent != subCluster || !got.Matched {
+ t.Errorf("hit d4 = %+v, want parent=%q matched=true", got, subCluster)
+ }
+ if got := byID["d2"]; got.Parent != rootCluster || !got.Matched {
+ t.Errorf("hit d2 = %+v, want parent=%q matched=true", got, rootCluster)
+ }
+ if got := byID["d3"]; got.Parent != "missing-cluster" || !got.Matched {
+ t.Errorf("orphan hit d3 = %+v, want it still returned with its edge", got)
+ }
+
+ // The ordinary (no-keyword) read is unchanged: depth-0 clusters only.
+ clusters, rootTotal, err := ns.ListClusters(t.Context(), "t1", "kb1", "", 0, 100)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if rootTotal != 1 || len(clusters) != 1 || clusters[0].Name != rootCluster {
+ t.Fatalf("root listing = %+v (total %d), want the single root cluster", clusters, rootTotal)
+ }
+ if clusters[0].Matched {
+ t.Error("a non-search read must not mark nodes as hits")
+ }
+}
+
+// TestSearchNavClusterCount pins the search header's number: how many clusters
+// the hits landed in — not the hit count (which would make the header mean
+// "documents" in search mode), and not every cluster row returned either, since
+// an ancestor shared by two hits is a path row.
+func TestSearchNavClusterCount(t *testing.T) {
+ hit := func(docID, parent string) map[string]interface{} {
+ return map[string]interface{}{"doc_id": docID, "parent_kwd": parent}
+ }
+ cases := []struct {
+ name string
+ matches []map[string]interface{}
+ want int64
+ }{
+ {
+ name: "no hits",
+ matches: nil,
+ want: 0,
+ },
+ {
+ name: "two hits in one cluster count once",
+ matches: []map[string]interface{}{hit("d1", "三国 11111111"), hit("d2", "三国 11111111")},
+ want: 1,
+ },
+ {
+ name: "two sub-clusters under one root count their own clusters, not the ancestor",
+ matches: []map[string]interface{}{
+ hit("d1", "名将 22222222"), hit("d2", "谋士 33333333"),
+ },
+ want: 2,
+ },
+ {
+ name: "hits in different root clusters",
+ matches: []map[string]interface{}{hit("d1", "三国 11111111"), hit("d2", "唐宋 44444444")},
+ want: 2,
+ },
+ {
+ name: "a leaf with no cluster edge is not counted",
+ matches: []map[string]interface{}{hit("d1", navRootParent), hit("d2", "")},
+ want: 0,
+ },
+ {
+ name: "an empty row is dropped",
+ matches: []map[string]interface{}{{}, hit("d1", "三国 11111111")},
+ want: 1,
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := searchNavClusterCount(tc.matches); got != tc.want {
+ t.Errorf("searchNavClusterCount = %d, want %d", got, tc.want)
+ }
+ })
+ }
}
// TestNavNamingHelpers_Readable verifies the nav display-name helpers produce
@@ -436,6 +645,48 @@ func TestNodeFromRow_ReadableName(t *testing.T) {
}
}
+// TestNodeFromRow_LabelSource pins which column the nav tree renders: title_kwd
+// (the summary's first line, the same label NavHit.Name carries to the model),
+// overridden by an explicit readable `name` column when the row has one (Python's
+// writer stores one; Go's rows do not). A cluster keeps its key from title_kwd.
+func TestNodeFromRow_LabelSource(t *testing.T) {
+ ns := &NavService{}
+ cases := []struct {
+ name string
+ row map[string]interface{}
+ want string
+ }{
+ {
+ name: "the content title labels a document leaf",
+ row: map[string]interface{}{
+ "title_kwd": "三国名将事迹概要", "type_kwd": "nav_doc",
+ },
+ want: "三国名将事迹概要",
+ },
+ {
+ name: "the name column wins over the content title",
+ row: map[string]interface{}{
+ "name": "三国名将事迹概要", "title_kwd": "机器键_ab12cd34ef56", "type_kwd": "nav_doc",
+ },
+ want: "三国名将事迹概要",
+ },
+ {
+ name: "a cluster keeps its key from title_kwd",
+ row: map[string]interface{}{
+ "title_kwd": "三国名将事迹概要 3fdf14d0", "type_kwd": "nav_cluster", "doc_count_int": 3,
+ },
+ want: "三国名将事迹概要 3fdf14d0",
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := ns.nodeFromRow(tc.row, "doc").Name; got != tc.want {
+ t.Errorf("nodeFromRow name = %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
+
// TestNavService_Search_ReturnsHit asserts acceptance #5.
func TestNavService_Search_ReturnsHit(t *testing.T) {
eng := newMemNavEngine()
@@ -706,6 +957,426 @@ func TestNavService_NavDocDepth(t *testing.T) {
}
}
+// TestNavService_NavDocLabel pins the nav_doc leaf's label on both write sites:
+// title_kwd carries the summary's first line, one column for both readers — the
+// tree (nodeFromRow) and the model (NavHit.Name → the routed document's
+// <summary>). No second label column is written.
+func TestNavService_NavDocLabel(t *testing.T) {
+ summary := "三国名将事迹概要\n三国名将事迹概述:关羽斩颜良诛文丑。"
+
+ t.Run("the leaf label is the summary's first line", func(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", summary)); err != nil {
+ t.Fatalf("UpsertDoc: %v", err)
+ }
+ if got := navDocRowTitle(t, eng, "d1"); got != "三国名将事迹概要" {
+ t.Errorf("nav_doc title_kwd = %q, want the summary's first line", got)
+ }
+ if _, ok := navDocRow(t, eng, "d1")["docnm_kwd"]; ok {
+ t.Error("nav_doc carries a separate docnm_kwd label; the label lives in title_kwd")
+ }
+ })
+
+ t.Run("the merge path labels the leaf too", func(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ // d1 creates the root cluster; d2's summary embeds identically, so it
+ // merges into that cluster (the second nav_doc write site).
+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", "aaa one")); err != nil {
+ t.Fatalf("UpsertDoc d1: %v", err)
+ }
+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d2", "aaa two")); err != nil {
+ t.Fatalf("UpsertDoc d2: %v", err)
+ }
+ if got := navDocRowTitle(t, eng, "d2"); got != "aaa two" {
+ t.Errorf("merged nav_doc title_kwd = %q, want the summary's first line", got)
+ }
+ })
+}
+
+// TestNavService_UpsertDoc_RestoresContentLabel covers the recompile upgrade for
+// legacy rows whose title_kwd holds the document file name (written by the
+// earlier label rule): the refresh puts the leaf back on the summary's first
+// line, in place (same row).
+func TestNavService_UpsertDoc_RestoresContentLabel(t *testing.T) {
+ summary := "三国名将事迹概要\n三国名将事迹概述:关羽斩颜良诛文丑。"
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ in := navUpsertInput("t1", "kb1", "d1", summary)
+ if err := ns.UpsertDoc(t.Context(), in); err != nil {
+ t.Fatalf("UpsertDoc: %v", err)
+ }
+ // Simulate a legacy row whose leaf label holds the file name.
+ navDocRow(t, eng, "d1")["title_kwd"] = "三国人物.pdf"
+
+ if err := ns.UpsertDoc(t.Context(), in); err != nil {
+ t.Fatalf("re-upsert: %v", err)
+ }
+ if got := navDocRowTitle(t, eng, "d1"); got != "三国名将事迹概要" {
+ t.Errorf("nav_doc title_kwd = %q, want it restored to the summary's first line", got)
+ }
+ assertSingleNavDocRow(t, eng, "d1")
+}
+
+// TestNavService_UpsertDoc_PlacementBySimilarity pins where a similar-but-not-
+// mergeable document opens its cluster, against the three similarity bands
+// (Python _MERGE_THRESHOLD=0.80 / _MIN_SIM=0.50):
+//
+// - >= 0.80 → merge into the best cluster (the document leaf joins it);
+// - < 0.80 → its own cluster, at the ROOT level when the best match is a
+// root cluster (Python attaches it to the best cluster's parent, which is
+// the root sentinel there), one level deeper when the best match is nested.
+//
+// The stub embedder's vectors are nearly collinear for any text, so the bands
+// are driven with explicit vectors (UpsertDocInput.Embedd) and seeded cluster
+// rows.
+func TestNavService_UpsertDoc_PlacementBySimilarity(t *testing.T) {
+ const dim = 1024
+
+ navVec := navTestUnitVec
+ seedCluster := func(eng *memNavEngine, name, parentKwd string, depth int, vec []float64) {
+ row := map[string]interface{}{
+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavCluster,
+ "title_kwd": name, "parent_kwd": parentKwd, "depth_int": depth,
+ "doc_count_int": 0, "content_with_weight": `{"type":"nav_cluster","description":"topic"}`,
+ "q_1024_vec": vec,
+ }
+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ findCluster := func(t *testing.T, eng *memNavEngine, parentKwd string) map[string]interface{} {
+ t.Helper()
+ for _, row := range eng.rows {
+ if row["type_kwd"] == nav.TypeNavCluster && row["parent_kwd"] == parentKwd {
+ return row
+ }
+ }
+ t.Fatalf("no nav_cluster row with parent_kwd=%q", parentKwd)
+ return nil
+ }
+ docRows := func(eng *memNavEngine, docID string) []map[string]interface{} {
+ out := []map[string]interface{}{}
+ for _, row := range eng.rows {
+ if row["type_kwd"] == nav.TypeNavDoc && row["doc_id"] == docID {
+ out = append(out, row)
+ }
+ }
+ return out
+ }
+
+ t.Run("an unrelated doc opens a root-level cluster", func(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ seedCluster(eng, "三国演义 aaaaaaaa", navRootParent, 0, navVec(1))
+
+ in := navUpsertInput("t1", "kb1", "d2", "出师表要义")
+ in.Embedd = f64ToF32Slice(navVec(0.60, 0.80)) // cos 0.60: related, not mergeable
+ if err := ns.UpsertDoc(t.Context(), in); err != nil {
+ t.Fatalf("UpsertDoc: %v", err)
+ }
+
+ // The seeded cluster keeps its name; the newly opened one is the other
+ // root-level cluster.
+ var opened map[string]interface{}
+ for _, row := range eng.rows {
+ if row["type_kwd"] == nav.TypeNavCluster && row["title_kwd"] != "三国演义 aaaaaaaa" {
+ opened = row
+ }
+ }
+ if opened == nil {
+ t.Fatal("no root-level cluster was opened for the unrelated document")
+ }
+ if opened["parent_kwd"] != navRootParent {
+ t.Errorf("new cluster parent_kwd = %v, want the root sentinel (top level, beside the existing cluster)", opened["parent_kwd"])
+ }
+ if d, _ := opened["depth_int"].(int); d != 0 {
+ t.Errorf("new cluster depth_int = %v, want 0 (a root cluster)", opened["depth_int"])
+ }
+ // Its document leaf sits one level below it.
+ rows := docRows(eng, "d2")
+ if len(rows) != 1 {
+ t.Fatalf("nav_doc rows for d2 = %d, want 1", len(rows))
+ }
+ if rows[0]["parent_kwd"] != opened["title_kwd"] {
+ t.Errorf("nav_doc parent = %v, want the new cluster %v", rows[0]["parent_kwd"], opened["title_kwd"])
+ }
+ if d, _ := rows[0]["depth_int"].(int); d != 1 {
+ t.Errorf("nav_doc depth_int = %v, want 1", rows[0]["depth_int"])
+ }
+ })
+
+ t.Run("a mergeable doc joins the best cluster", func(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ seedCluster(eng, "三国演义 aaaaaaaa", navRootParent, 0, navVec(1))
+
+ in := navUpsertInput("t1", "kb1", "d2", "三国名将")
+ in.Embedd = f64ToF32Slice(navVec(0.90, math.Sqrt(1-0.81))) // cos 0.90
+ if err := ns.UpsertDoc(t.Context(), in); err != nil {
+ t.Fatalf("UpsertDoc: %v", err)
+ }
+ rows := docRows(eng, "d2")
+ if len(rows) != 1 || rows[0]["parent_kwd"] != "三国演义 aaaaaaaa" {
+ t.Fatalf("nav_doc rows = %+v, want the leaf under the merged cluster", rows)
+ }
+ if clusters := countNavClusters(eng); clusters != 1 {
+ t.Errorf("nav_cluster rows = %d, want 1 (a merge must not open a cluster)", clusters)
+ }
+ })
+
+ t.Run("a nested best match keeps the child cluster", func(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ // Root cluster at cos 0.70 with the doc (>= _RECURSE_THRESHOLD, so the
+ // descent continues), nested cluster at 0.78 (< merge threshold, > the
+ // root's, so IT becomes the best match).
+ seedCluster(eng, "三国演义 aaaaaaaa", navRootParent, 0, navVec(1))
+ nested := "出师表要义 bbbbbbbb"
+ seedCluster(eng, nested, "三国演义 aaaaaaaa", 1, navVec(0.546, 0.5569, 0.6257))
+
+ in := navUpsertInput("t1", "kb1", "d3", "出师表陈情")
+ in.Embedd = f64ToF32Slice(navVec(0.70, 0.714)) // cos(root) 0.70, cos(child) 0.78
+ if err := ns.UpsertDoc(t.Context(), in); err != nil {
+ t.Fatalf("UpsertDoc: %v", err)
+ }
+ child := findCluster(t, eng, nested)
+ if d, _ := child["depth_int"].(int); d != 2 {
+ t.Errorf("new child cluster depth_int = %v, want 2 (best depth 1 + 1)", child["depth_int"])
+ }
+ rows := docRows(eng, "d3")
+ if len(rows) != 1 || rows[0]["parent_kwd"] != child["title_kwd"] {
+ t.Fatalf("nav_doc rows = %+v, want the leaf under the new child cluster %v", rows, child["title_kwd"])
+ }
+ if d, _ := rows[0]["depth_int"].(int); d != 3 {
+ t.Errorf("nav_doc depth_int = %v, want 3", rows[0]["depth_int"])
+ }
+ })
+
+}
+
+// navTestUnitVec builds a 1024-dim unit vector from the leading components (the
+// rest zero), so a test can dial an exact cosine against a seeded cluster's
+// vector. 1024 is the stub embedder's dimension.
+func navTestUnitVec(components ...float64) []float64 {
+ v := make([]float64, 1024)
+ var norm float64
+ for i, c := range components {
+ v[i] = c
+ norm += c * c
+ }
+ if norm > 0 {
+ scale := 1 / math.Sqrt(norm)
+ for i := range v {
+ v[i] *= scale
+ }
+ }
+ return v
+}
+
+// TestNavService_FindBestCluster_DeepestMatchWins pins the descent result against
+// Python _find_best_cluster: the returned match is the DEEPEST cluster reached
+// while every step clears navRecurse (0.65) — not the strongest one seen — and a
+// child below the threshold is not adopted at all. The placement branch decides
+// from that (deeper, possibly lower) similarity, so this is what keeps the two
+// implementations in step.
+func TestNavService_FindBestCluster_DeepestMatchWins(t *testing.T) {
+ const (
+ rootName = "三国演义 aaaaaaaa"
+ childName = "出师表要义 bbbbbbbb"
+ )
+ seedCluster := func(eng *memNavEngine, name, parentKwd string, depth int, vec []float64) {
+ row := map[string]interface{}{
+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavCluster,
+ "title_kwd": name, "parent_kwd": parentKwd, "depth_int": depth,
+ "doc_count_int": 0, "content_with_weight": `{"type":"nav_cluster","description":"topic"}`,
+ "q_1024_vec": vec,
+ }
+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ // doc/child pairs are built so the document's cosine against each seeded
+ // cluster vector is exactly the intended value.
+ cases := []struct {
+ name string
+ doc []float64
+ child []float64
+ wantName string
+ wantSim float64
+ wantDepth int
+ }{
+ {
+ // Root 0.85 (mergeable on its own) but the child still clears the
+ // recurse threshold at 0.70: Python descends and returns the CHILD,
+ // so the placement sees 0.70 and opens a new cluster instead of
+ // merging into the root.
+ name: "a recurseable descendant displaces a stronger ancestor",
+ doc: navTestUnitVec(0.85, 0.5268),
+ child: navTestUnitVec(0.595, 0.3688, 0.7141),
+ wantName: childName,
+ wantSim: 0.70,
+ wantDepth: 1,
+ },
+ {
+ // The child is below the threshold: it is not adopted at all, so the
+ // root (0.90) stays the answer and the document merges into it.
+ name: "a child below the recurse threshold is not adopted",
+ doc: navTestUnitVec(0.90, 0.4359),
+ child: navTestUnitVec(0.36, 0.1744, 0.9165),
+ wantName: rootName,
+ wantSim: 0.90,
+ wantDepth: 0,
+ },
+ {
+ // The root itself is returned even when it does not clear the
+ // threshold (Python seeds `best` with its top-1 root).
+ name: "an unrelated root is still returned",
+ doc: navTestUnitVec(0.40, 0.9165),
+ child: navTestUnitVec(0.36, 0.1744, 0.9165),
+ wantName: rootName,
+ wantSim: 0.40,
+ wantDepth: 0,
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ seedCluster(eng, rootName, navRootParent, 0, navTestUnitVec(1))
+ seedCluster(eng, childName, rootName, 1, tc.child)
+
+ name, sim, depth, err := ns.findBestCluster(t.Context(), "t1", "kb1", f64ToF32Slice(tc.doc))
+ if err != nil {
+ t.Fatalf("findBestCluster: %v", err)
+ }
+ if name != tc.wantName || depth != tc.wantDepth {
+ t.Errorf("findBestCluster = (%q, %v, %d), want (%q, %v, %d)",
+ name, sim, depth, tc.wantName, tc.wantSim, tc.wantDepth)
+ }
+ if math.Abs(sim-tc.wantSim) > 1e-4 {
+ t.Errorf("findBestCluster sim = %v, want %v", sim, tc.wantSim)
+ }
+ })
+ }
+}
+
+// countNavClusters counts the nav_cluster rows of the mem engine.
+func countNavClusters(eng *memNavEngine) int {
+ n := 0
+ for _, row := range eng.rows {
+ if row["type_kwd"] == nav.TypeNavCluster {
+ n++
+ }
+ }
+ return n
+}
+
+// f64ToF32Slice narrows an explicit test vector to the engine's float32 shape.
+func f64ToF32Slice(in []float64) []float32 {
+ out := make([]float32, len(in))
+ for i, v := range in {
+ out[i] = float32(v)
+ }
+ return out
+}
+
+// assertSingleNavDocRow fails when docID has anything other than one nav_doc
+// row: the label refresh must UPDATE, never duplicate the leaf.
+func assertSingleNavDocRow(t *testing.T, eng *memNavEngine, docID string) {
+ t.Helper()
+ count := 0
+ for _, row := range eng.rows {
+ if row["type_kwd"] == "nav_doc" && row["doc_id"] == docID {
+ count++
+ }
+ }
+ if count != 1 {
+ t.Errorf("nav_doc rows for %s = %d, want 1 (refresh, not a duplicate)", docID, count)
+ }
+}
+
+// TestNavService_SummariesByDocIDs_DescriptionLabel pins the routed-document
+// label — which callers render to the model as the document's <summary> — to the
+// nav_doc's stored description, i.e. the document's own summary text (Python
+// _nav_doc_summaries). The row's label is only the fallback: title_kwd carries
+// the summary's first line, not the whole text.
+func TestNavService_SummariesByDocIDs_DescriptionLabel(t *testing.T) {
+ summary := "三国名将事迹概要\n三国名将事迹概述:关羽斩颜良诛文丑。"
+
+ t.Run("the description wins over the tree label", func(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", summary)); err != nil {
+ t.Fatalf("UpsertDoc: %v", err)
+ }
+ got := ns.SummariesByDocIDs(t.Context(), "t1", "kb1", []string{"d1"})
+ if got["d1"] != summary {
+ t.Errorf("routed label = %q, want the stored description", got["d1"])
+ }
+ if got["d1"] == navDocRowTitle(t, eng, "d1") {
+ t.Error("routed label fell back to the tree label; the description must win")
+ }
+ })
+
+ t.Run("a row without a description falls back to its label", func(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{
+ {
+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,
+ "doc_id": "d9", "title_kwd": "出师表全文内容与赏析",
+ "content_with_weight": `{"type":"nav_doc"}`,
+ },
+ }, "", "kb1"); err != nil {
+ t.Fatal(err)
+ }
+ got := ns.SummariesByDocIDs(t.Context(), "t1", "kb1", []string{"d9"})
+ if got["d9"] != "出师表全文内容与赏析" {
+ t.Errorf("label = %q, want the stored title_kwd fallback", got["d9"])
+ }
+ })
+
+ t.Run("a raw id is never a label", func(t *testing.T) {
+ const rawID = "6564be8a6564be8a6564be8a6564be8a"
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{
+ {
+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,
+ "doc_id": rawID, "title_kwd": rawID,
+ "content_with_weight": `{"type":"nav_doc"}`,
+ },
+ }, "", "kb1"); err != nil {
+ t.Fatal(err)
+ }
+ got := ns.SummariesByDocIDs(t.Context(), "t1", "kb1", []string{rawID})
+ if _, ok := got[rawID]; ok {
+ t.Errorf("raw id used as a label: %+v", got)
+ }
+ })
+}
+
+// navDocRow returns the nav_doc row for docID.
+func navDocRow(t *testing.T, eng *memNavEngine, docID string) map[string]interface{} {
+ t.Helper()
+ for _, row := range eng.rows {
+ if row["type_kwd"] == "nav_doc" && row["doc_id"] == docID {
+ return row
+ }
+ }
+ t.Fatalf("no nav_doc row for doc %s", docID)
+ return nil
+}
+
+// navDocRowTitle returns the title_kwd of the nav_doc row for docID.
+func navDocRowTitle(t *testing.T, eng *memNavEngine, docID string) string {
+ t.Helper()
+ title, _ := navDocRow(t, eng, docID)["title_kwd"].(string)
+ return title
+}
+
// TestNavService_RemoveDoc_CascadesToEmptyCluster covers A3: after removing a
// doc, a cluster left with no docs and no children is pruned (cleanupEmptyCluster),
// and the doc is dropped from its parent cluster's doc_ids_kwd.
@@ -847,6 +1518,159 @@ func TestNavService_MaybeSplitCluster_SplitsOverfull(t *testing.T) {
}
}
+// stubNavMergeLLM is a canned nav.NavMergeLLM: it names every new cluster with
+// the configured topic title and fuses merges into the configured text.
+type stubNavMergeLLM struct {
+ name string
+ summary string
+ merged string
+ mergeSeen [][]string
+}
+
+func (s *stubNavMergeLLM) CreateSummary(_ context.Context, _, _ string) (string, string, error) {
+ return s.name, s.summary, nil
+}
+
+func (s *stubNavMergeLLM) Merge(_ context.Context, _ string, texts []string) (string, error) {
+ s.mergeSeen = append(s.mergeSeen, texts)
+ return s.merged, nil
+}
+
+// recordingNavEmbedder records every embedded text so a test can assert that a
+// rewritten cluster description was re-embedded.
+type recordingNavEmbedder struct {
+ texts []string
+}
+
+func (e *recordingNavEmbedder) Encode(ctx context.Context, tenantID string, texts []string) ([][]float32, error) {
+ e.texts = append(e.texts, texts...)
+ return stubNavEmbedder{}.Encode(ctx, tenantID, texts)
+}
+
+// navClusterRow returns the first nav_cluster row of the mem engine.
+func navClusterRow(t *testing.T, eng *memNavEngine) map[string]interface{} {
+ t.Helper()
+ for _, row := range eng.rows {
+ if row["type_kwd"] == "nav_cluster" {
+ return row
+ }
+ }
+ t.Fatal("no nav_cluster row")
+ return nil
+}
+
+// navRowDescription parses a nav row payload's description.
+func navRowDescription(t *testing.T, row map[string]interface{}) string {
+ t.Helper()
+ raw, _ := row["content_with_weight"].(string)
+ var m map[string]interface{}
+ if err := json.Unmarshal([]byte(raw), &m); err != nil {
+ t.Fatalf("parse nav payload %q: %v", raw, err)
+ }
+ desc, _ := m["description"].(string)
+ return desc
+}
+
+// TestNavService_LLMClusterNaming pins the wired naming path: the cluster takes
+// the model's topic title as the title part and keeps the " <8-hex>" suffix —
+// the name is also the cluster key (parent_kwd / row id), so it must stay unique
+// per KB — and the model's summary becomes the cluster's description.
+func TestNavService_LLMClusterNaming(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ ns.SetNavMergeLLM(&stubNavMergeLLM{name: "三国名将事迹", summary: "三国名将事迹概览。"})
+ summary := "三国名将事迹概要\n正文"
+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", summary)); err != nil {
+ t.Fatalf("UpsertDoc: %v", err)
+ }
+ cluster := navClusterRow(t, eng)
+ if want := readableClusterName("三国名将事迹", summary); cluster["title_kwd"] != want {
+ t.Errorf("cluster title_kwd = %v, want %q (topic title + uniqueness suffix)", cluster["title_kwd"], want)
+ }
+ if got := navRowDescription(t, cluster); got != "三国名将事迹概览。" {
+ t.Errorf("cluster description = %q, want the model summary", got)
+ }
+}
+
+// TestNavService_MergeFusesClusterDescription pins the wired merge path: a
+// document that joins an existing cluster fuses its summary into the cluster
+// description (Python _llm_merge) and re-embeds the rewritten description, so
+// later placements score against the merged topic.
+func TestNavService_MergeFusesClusterDescription(t *testing.T) {
+ eng := newMemNavEngine()
+ embedder := &recordingNavEmbedder{}
+ ns := NewNavService(embedder)
+ ns.engine = eng
+ stub := &stubNavMergeLLM{name: "topic", summary: "first doc summary", merged: "fused description"}
+ ns.SetNavMergeLLM(stub)
+
+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", "aaa one")); err != nil {
+ t.Fatalf("UpsertDoc d1: %v", err)
+ }
+ // d2's summary embeds identically under the deterministic stub embedder, so
+ // it is placed inside d1's cluster (the merge branch).
+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d2", "aaa two")); err != nil {
+ t.Fatalf("UpsertDoc d2: %v", err)
+ }
+
+ clusters := 0
+ for _, row := range eng.rows {
+ if row["type_kwd"] == "nav_cluster" {
+ clusters++
+ }
+ }
+ if clusters != 1 {
+ t.Fatalf("nav_cluster rows = %d, want 1 (d2 must merge, not create a cluster)", clusters)
+ }
+ cluster := navClusterRow(t, eng)
+ if got := navRowDescription(t, cluster); got != "fused description" {
+ t.Errorf("cluster description = %q, want the merged text", got)
+ }
+ if ids := firstStringSlice(cluster["doc_ids_kwd"]); len(ids) != 2 {
+ t.Errorf("cluster doc_ids_kwd = %v, want both documents", ids)
+ }
+ if len(stub.mergeSeen) != 1 {
+ t.Fatalf("Merge calls = %d, want 1", len(stub.mergeSeen))
+ }
+ if texts := stub.mergeSeen[0]; len(texts) != 2 || texts[0] != "first doc summary" || texts[1] != "aaa two" {
+ t.Errorf("Merge texts = %v, want [existing description, new summary]", texts)
+ }
+ reEmbedded := false
+ for _, text := range embedder.texts {
+ if text == "fused description" {
+ reEmbedded = true
+ break
+ }
+ }
+ if !reEmbedded {
+ t.Errorf("merged description was not re-embedded; embedded texts = %v", embedder.texts)
+ }
+}
+
+// TestNavService_MergeKeepsDescriptionWithoutLLM pins the no-LLM fallback:
+// without a NavMergeLLM the cluster keeps the description it was created with
+// (Python _llm_merge returns the existing text when chat_mdl is nil) instead of
+// accumulating a concatenation of every document summary.
+func TestNavService_MergeKeepsDescriptionWithoutLLM(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", "aaa one")); err != nil {
+ t.Fatalf("UpsertDoc d1: %v", err)
+ }
+ cluster := navClusterRow(t, eng)
+ before := navRowDescription(t, cluster)
+
+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d2", "aaa two")); err != nil {
+ t.Fatalf("UpsertDoc d2: %v", err)
+ }
+ if got := navRowDescription(t, cluster); got != before {
+ t.Errorf("cluster description = %q, want it unchanged (%q) without an LLM", got, before)
+ }
+ if ids := firstStringSlice(cluster["doc_ids_kwd"]); len(ids) != 2 {
+ t.Errorf("cluster doc_ids_kwd = %v, want both documents", ids)
+ }
+}
+
// sTestNavIndex returns the nav index name for a test NavService (tenant "t1").
func sTestNavIndex(ns *NavService) string {
return "ragflow_t1"
diff --git a/web/src/interfaces/database/dataset-nav.ts b/web/src/interfaces/database/dataset-nav.ts
index 7916a2b995..69f45743da 100644
--- a/web/src/interfaces/database/dataset-nav.ts
+++ b/web/src/interfaces/database/dataset-nav.ts
@@ -8,9 +8,22 @@ export interface DatasetNavNode {
keywords?: string[];
entities?: string[];
graph_content?: string;
+ /**
+ * Tree edge: the parent cluster's name ("root" for a depth-0 cluster). A
+ * keyword search returns the hits together with the cluster path above them,
+ * so the flat list nests into one tree per root cluster.
+ */
+ parent_kwd?: string;
+ /** Keyword-search hit marker; the path rows returned with it are not hits. */
+ matched?: boolean;
}
export interface DatasetNavList {
+ /**
+ * Cluster count: the depth-0 clusters when browsing, or the clusters the hits
+ * landed in when searching. The nav tree header shows it in both modes, so a
+ * search never re-purposes that heading as a document count.
+ */
total: number;
items: DatasetNavNode[];
}
diff --git a/web/src/pages/dataset/compilation/hooks/use-compilation-nav.ts b/web/src/pages/dataset/compilation/hooks/use-compilation-nav.ts
index 82c35e8e0f..90011d3274 100644
--- a/web/src/pages/dataset/compilation/hooks/use-compilation-nav.ts
+++ b/web/src/pages/dataset/compilation/hooks/use-compilation-nav.ts
@@ -66,12 +66,11 @@ export function useCompilationNav() {
const { data: childrenData, isError: childrenError } =
useFetchDatasetNavChildren(loadingParent, activeKeywords);
+ // Opened documents get their FULL structure graph, without the nav keywords: a
+ // keyword-filtered read would hide every entity that does not match (keyword
+ // drill-down lives in the structure view's own search box).
const { data: structureData, isPlaceholderData: structurePlaceholder } =
- useFetchDocumentStructureGraphById(
- kbId,
- loadingDocId ?? '',
- activeKeywords || undefined,
- );
+ useFetchDocumentStructureGraphById(kbId, loadingDocId ?? '');
useEffect(() => {
if (!loadingParent || !childrenData) {
diff --git a/web/src/pages/dataset/compilation/nav-tree-left-panel.tsx b/web/src/pages/dataset/compilation/nav-tree-left-panel.tsx
index f725ba476d..d27779c156 100644
--- a/web/src/pages/dataset/compilation/nav-tree-left-panel.tsx
+++ b/web/src/pages/dataset/compilation/nav-tree-left-panel.tsx
@@ -18,6 +18,12 @@ import { useTranslation } from 'react-i18next';
import { UpdateLogSheet } from './update-log-sheet';
import { buildNavTreeData, NavEntityClickHandler } from './utils/nav-tree';
+// TreeView only computes expandedItemIds when initialSelectedItemId is truthy;
+// combined with expandAll, any truthy id makes every branch mount open. A
+// sentinel that matches no real node forces expand-all without highlighting any
+// row as selected (same trick as skills-left-panel).
+const NavExpandAllSentinelId = '__nav-tree-expand-all-sentinel__';
+
type NavNodeDeleteActionProps = {
name: string;
parentName: string | null;
@@ -152,6 +158,10 @@ export function NavTreeLeftPanel({
childrenMap,
childrenErrorParents,
structureMap,
+ // A search response is a pruned forest (hits + the cluster path above
+ // them), so it is nested from the payload instead of being fetched
+ // branch by branch.
+ searchMode: !!activeKeywords,
getActions: renderNavActions,
onNodeClick,
onNodeExpand,
@@ -161,6 +171,7 @@ export function NavTreeLeftPanel({
}),
[
navList?.items,
+ activeKeywords,
childrenMap,
childrenErrorParents,
structureMap,
@@ -244,6 +255,11 @@ export function NavTreeLeftPanel({
<TreeView
key={activeKeywords}
data={treeData}
+ // Search: mount the matched branches open (sentinel trick above).
+ expandAll={!!activeKeywords}
+ initialSelectedItemId={
+ activeKeywords ? NavExpandAllSentinelId : undefined
+ }
expandOnRowClick={false}
defaultNodeIcon={Folder}
defaultLeafIcon={FileText}
diff --git a/web/src/pages/dataset/compilation/utils/nav-tree.test.ts b/web/src/pages/dataset/compilation/utils/nav-tree.test.ts
index 7ff388f7aa..e7d2c8af68 100644
--- a/web/src/pages/dataset/compilation/utils/nav-tree.test.ts
+++ b/web/src/pages/dataset/compilation/utils/nav-tree.test.ts
@@ -1,4 +1,7 @@
-import { buildNavTreeData } from './nav-tree';
+import { adaptTreeToTreeData } from '@/components/structure-graph/adapters';
+import { CompilationTemplateKind } from '@/constants/compilation';
+
+import { buildNavTreeData, nestNavSearchHits } from './nav-tree';
jest.mock('@/components/structure-graph/adapters', () => ({
adaptPageIndexToTreeData: jest.fn(() => []),
@@ -46,3 +49,169 @@ describe('buildNavTreeData', () => {
expect(data[0].children).toBeUndefined();
});
});
+
+function doc(docId: string, name: string, parentKwd: string, matched = true) {
+ return {
+ name,
+ description: '',
+ doc_count: 1,
+ type: 'doc',
+ doc_id: docId,
+ has_children: false,
+ parent_kwd: parentKwd,
+ matched,
+ };
+}
+
+function clusterIn(name: string, parentKwd: string, docCount = 1) {
+ return {
+ name,
+ description: '',
+ doc_count: docCount,
+ type: 'cluster',
+ has_children: true,
+ parent_kwd: parentKwd,
+ matched: false,
+ };
+}
+
+describe('nestNavSearchHits', () => {
+ it('turns the flat hits plus their cluster path into a forest', () => {
+ const { roots, children } = nestNavSearchHits([
+ clusterIn('三国 11111111', 'root'),
+ clusterIn('名将 22222222', '三国 11111111'),
+ doc('d1', '三国人物.pdf', '名将 22222222'),
+ doc('d2', '其他.pdf', '三国 11111111'),
+ ]);
+
+ expect(roots.map((node) => node.name)).toEqual(['三国 11111111']);
+ expect(children['cluster:三国 11111111'].map((node) => node.name)).toEqual([
+ '名将 22222222',
+ '其他.pdf',
+ ]);
+ expect(children['cluster:名将 22222222'].map((node) => node.name)).toEqual([
+ '三国人物.pdf',
+ ]);
+ });
+
+ it('keeps a hit whose parent is not in the result set as its own root', () => {
+ const { roots, children } = nestNavSearchHits([
+ doc('d9', '孤儿.pdf', 'missing-cluster'),
+ ]);
+
+ expect(roots.map((node) => node.name)).toEqual(['孤儿.pdf']);
+ expect(children).toEqual({});
+ });
+});
+
+describe('buildNavTreeData (search mode)', () => {
+ const SearchOptions = { ...TreeOptions, childrenMap: {}, searchMode: true };
+
+ it('renders ONE tree per root cluster holding only the matched branches', () => {
+ const data = buildNavTreeData(
+ [
+ clusterIn('三国 11111111', 'root', 2),
+ clusterIn('名将 22222222', '三国 11111111'),
+ doc('d1', '三国人物.pdf', '名将 22222222'),
+ doc('d2', '其他.pdf', '三国 11111111'),
+ ],
+ SearchOptions,
+ );
+
+ expect(data).toHaveLength(1);
+ expect(data[0].id).toBe('cluster:三国 11111111');
+ expect(data[0].hasChildren).toBe(true);
+ // Child ids are prefixed with the parent's id so they stay unique across
+ // the forest.
+ expect(data[0].children?.map((child) => child.id)).toEqual([
+ 'cluster:三国 11111111/cluster:名将 22222222',
+ 'cluster:三国 11111111/doc:d2',
+ ]);
+ // The deeper match sits under its matched ancestor, not beside it.
+ expect(data[0].children?.[0].children?.map((child) => child.id)).toEqual([
+ 'cluster:三国 11111111/cluster:名将 22222222/doc:d1',
+ ]);
+ });
+
+ it('does not mount a pending structure placeholder under a matched leaf', () => {
+ // The branches mount already expanded, so a placeholder would never be
+ // replaced (mount-time expansion fires no onExpand, which fetches the graph).
+ const data = buildNavTreeData([doc('d1', '三国人物.pdf', 'root')], {
+ ...SearchOptions,
+ structureMap: {},
+ });
+
+ expect(data).toHaveLength(1);
+ expect(data[0].children).toBeUndefined();
+ expect(data[0].onExpand).toEqual(expect.any(Function));
+ });
+
+ it('mounts the fetched structure graph under a matched leaf', () => {
+ jest
+ .mocked(adaptTreeToTreeData)
+ .mockReturnValueOnce([
+ { id: 'e1', name: '刘备', children: [{ id: 'e2', name: '关羽' }] },
+ ]);
+ const onEntityClick = jest.fn();
+ const data = buildNavTreeData([doc('d1', '三国人物.pdf', 'root')], {
+ ...SearchOptions,
+ onEntityClick,
+ structureMap: {
+ d1: [
+ {
+ template_id: 't1',
+ template_name: '结构树',
+ kind: CompilationTemplateKind.Tree,
+ entities: [
+ { id: 'e1', name: '刘备', description: '蜀汉开国皇帝' },
+ { id: 'e2', name: '关羽' },
+ ],
+ relations: [],
+ },
+ ],
+ },
+ });
+
+ expect(data).toHaveLength(1);
+ expect(data[0].hasChildren).toBe(true);
+ // Adapter ids are re-prefixed as <document id>/<template id>/<entity id>, so
+ // they stay unique across the forest, and the hit stays expandable.
+ expect(data[0].children?.map((child) => child.id)).toEqual([
+ 'doc:d1/t1/e1',
+ ]);
+ expect(data[0].children?.[0].children?.map((child) => child.id)).toEqual([
+ 'doc:d1/t1/e2',
+ ]);
+ expect(data[0].onExpand).toEqual(expect.any(Function));
+ // Clicking an entity reports its document, display name and description —
+ // the same payload the non-search tree sends.
+ data[0].children?.[0].onClick?.();
+ expect(onEntityClick).toHaveBeenCalledWith(
+ expect.objectContaining({ doc_id: 'd1' }),
+ '刘备',
+ '蜀汉开国皇帝',
+ );
+ });
+
+ it('leaves the matched leaf a leaf when the fetched graph holds no entity', () => {
+ // The adapter is mocked to return no items, so a fetched-but-empty graph
+ // renders no children instead of an empty expandable branch.
+ const data = buildNavTreeData([doc('d1', '三国人物.pdf', 'root')], {
+ ...SearchOptions,
+ structureMap: {
+ d1: [
+ {
+ template_id: 't1',
+ template_name: '结构树',
+ kind: CompilationTemplateKind.Tree,
+ entities: [],
+ relations: [],
+ },
+ ],
+ },
+ });
+
+ expect(data[0].children).toBeUndefined();
+ expect(data[0].onExpand).toEqual(expect.any(Function));
+ });
+});
diff --git a/web/src/pages/dataset/compilation/utils/nav-tree.ts b/web/src/pages/dataset/compilation/utils/nav-tree.ts
index 923d79ca39..80e11317e9 100644
--- a/web/src/pages/dataset/compilation/utils/nav-tree.ts
+++ b/web/src/pages/dataset/compilation/utils/nav-tree.ts
@@ -28,6 +28,12 @@ type BuildNavTreeDataOptions = {
childrenMap: Record<string, DatasetNavNode[]>;
childrenErrorParents?: Record<string, boolean>;
structureMap: Record<string, IStructureGraphTemplate[]>;
+ /**
+ * Search mode: the items are a pruned forest — the matched document leaves
+ * plus the cluster path above each of them, every row carrying parent_kwd —
+ * so branches come from the payload instead of a lazy /children request.
+ */
+ searchMode?: boolean;
getActions?: NavTreeActionsFactory;
onNodeClick: (node: DatasetNavNode, parentName: string | null) => void;
onNodeExpand: (node: DatasetNavNode) => void;
@@ -117,20 +123,153 @@ function buildStructureTreeData(
});
}
+// navNodeIdentity is the forest's node key. Identity, not display name: two
+// documents may share a name, and a cluster's display name is exactly the key
+// its children reference through parent_kwd.
+function navNodeIdentity(node: DatasetNavNode): string {
+ return node.type === 'doc' && node.doc_id
+ ? `doc:${node.doc_id}`
+ : `cluster:${node.name}`;
+}
+
+type NavSearchForest = {
+ roots: DatasetNavNode[];
+ children: Record<string, DatasetNavNode[]>;
+};
+
+// nestNavSearchHits groups a search response into the tree its rows describe: a
+// row whose parent edge points at another returned row becomes that row's child,
+// everything else is a root. The backend returns every hit together with the
+// cluster path above it, so this yields ONE tree per root cluster holding only
+// the matched branches — instead of one top-level tree per hit, where a matching
+// cluster's own matching subtrees showed up as trees beside it.
+export function nestNavSearchHits(
+ items: DatasetNavNode[] = [],
+): NavSearchForest {
+ const present = new Set(items.map(navNodeIdentity));
+ const children: Record<string, DatasetNavNode[]> = {};
+ const roots: DatasetNavNode[] = [];
+ items.forEach((node) => {
+ const parentKey = node.parent_kwd ? `cluster:${node.parent_kwd}` : '';
+ if (parentKey && present.has(parentKey)) {
+ const bucket = children[parentKey];
+ if (bucket) {
+ bucket.push(node);
+ } else {
+ children[parentKey] = [node];
+ }
+ } else {
+ roots.push(node);
+ }
+ });
+ return { roots, children };
+}
+
+// mountDocumentStructure attaches a document leaf's structure-graph children to
+// its tree item, or a loading placeholder while the graph is being fetched.
+//
+// The search path passes pendingPlaceholder false: its branches mount already
+// expanded (expandAll), so a pending row under every hit would sit there for
+// good — the mount-time expansion never fires onExpand, which is what triggers
+// the fetch.
+function mountDocumentStructure(
+ item: TreeDataItem,
+ node: DatasetNavNode,
+ id: string,
+ options: BuildNavTreeDataOptions,
+ { pendingPlaceholder = true } = {},
+): void {
+ const templates = options.structureMap[node.doc_id as string];
+ if (!templates) {
+ // Not fetched yet: the placeholder keeps the node rendered as an expandable
+ // branch until the request resolves.
+ item.hasChildren = true;
+ if (pendingPlaceholder) {
+ item.children = [
+ { id: `${id}/__loading__`, name: options.loadingPlaceholder },
+ ];
+ }
+ return;
+ }
+ const children = buildStructureTreeData(templates, id, (entity) =>
+ options.onEntityClick?.(
+ node,
+ getEntityDisplayName(entity),
+ getEntityDescription(entity),
+ ),
+ );
+ if (children.length) {
+ item.hasChildren = true;
+ item.children = children;
+ }
+ // No entity nodes: leave hasChildren unset so the node stays a leaf.
+}
+
+// buildNavSearchTreeData renders the pruned search forest. Branches come from
+// the response, so expanding one fetches nothing; a matched document leaf still
+// expands into its structure graph.
+function buildNavSearchTreeData(
+ nodes: DatasetNavNode[],
+ forest: NavSearchForest,
+ options: BuildNavTreeDataOptions,
+ parentName: string | null,
+ idPrefix: string,
+): TreeDataItem[] {
+ const { getActions, onNodeClick, onNodeExpand } = options;
+ return nodes.map((node) => {
+ const identity = navNodeIdentity(node);
+ const id = idPrefix ? `${idPrefix}/${identity}` : identity;
+ const item: TreeDataItem = {
+ id,
+ name: trim(node.name),
+ actions: getActions?.(node, parentName),
+ onClick: () => onNodeClick(node, parentName),
+ };
+
+ const branches = forest.children[identity];
+ if (branches?.length) {
+ item.hasChildren = true;
+ item.children = buildNavSearchTreeData(
+ branches,
+ forest,
+ options,
+ node.name,
+ id,
+ );
+ } else if (node.doc_id) {
+ item.onExpand = () => onNodeExpand(node);
+ mountDocumentStructure(item, node, id, options, {
+ pendingPlaceholder: false,
+ });
+ }
+
+ return item;
+ });
+}
+
export function buildNavTreeData(
items: DatasetNavNode[] = [],
options: BuildNavTreeDataOptions,
parentName: string | null = null,
idPrefix = '',
): TreeDataItem[] {
+ if (options.searchMode) {
+ const forest = nestNavSearchHits(items);
+ return buildNavSearchTreeData(
+ forest.roots,
+ forest,
+ options,
+ parentName,
+ idPrefix,
+ );
+ }
+
const {
childrenMap,
childrenErrorParents,
- structureMap,
getActions,
onNodeClick,
onNodeExpand,
- onEntityClick,
loadingPlaceholder,
errorPlaceholder,
} = options;
@@ -160,26 +299,7 @@ export function buildNavTreeData(
// Fetched but empty: leave children unset so the branch opens to nothing.
} else if (node.doc_id) {
// Document leaf: expandable into its structure graph entities.
- const templates = structureMap[node.doc_id];
- if (!templates) {
- // Not fetched yet: a placeholder keeps the node rendered as an
- // expandable branch until the request resolves.
- item.hasChildren = true;
- item.children = [{ id: `${id}/__loading__`, name: loadingPlaceholder }];
- } else {
- const children = buildStructureTreeData(templates, id, (entity) =>
- onEntityClick?.(
- node,
- getEntityDisplayName(entity),
- getEntityDescription(entity),
- ),
- );
- if (children.length) {
- item.hasChildren = true;
- item.children = children;
- }
- // No entity nodes: leave hasChildren unset so the node stays a leaf.
- }
+ mountDocumentStructure(item, node, id, options);
}
return item;