RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(nav): navigation tree search, LLM cluster naming, cluster depth (#20188)
qinling0210ab247dc50894
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex 29613c56a5..49bb9dfda5 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -818,7 +818,16 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, serverName stri// unavailable, skipping dataset-nav upsert"), leaving the dataset tree empty.// The embedder resolves the tenant's embedding model on demand, so both// Search and UpsertDoc can embed queries/summaries automatically.- nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(service.NewModelProviderService(), "")))+ navModelService := service.NewModelProviderService()+ navService := nlp.NewNavService(service.NewNavEmbedder(navModelService, ""))+ // The cluster namer/merger is the counterpart of Python's+ // _llm_create_summary / _llm_merge: a new cluster is named after the model's+ // topic title and an existing cluster's description is fused with the joining+ // document. Passing an empty model ref resolves each tenant's default chat+ // model on demand; without one the nav service keeps its deterministic+ // behaviour.+ navService.SetNavMergeLLM(service.NewNavMergeLLM(navModelService, ""))+ nav.SetNavService(navService)// Memory extraction runs on the Ingestor's shared NATS consumer + worker// pool (task_type="memory" dispatched by handleAndExecute -> executeMemoryTask),// so there is no longer a dedicated Redis memory consumer to start.@@ -1141,8 +1150,12 @@ func startServer(ctx context.Context, serverName string, arguments *serverArgs)// Install the dataset-nav ES-backed service (internal/service/nav +// internal/service/nlp). The embedder resolves the tenant's embedding model- // on demand so Search/UpsertDoc can embed queries/summaries automatically.- nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(modelProviderService, "")))+ // on demand so Search/UpsertDoc can embed queries/summaries automatically,+ // and the cluster namer/merger resolves the tenant's default chat model the+ // same way (see the ingestor's wrapping of SetNavMergeLLM).+ apiNavService := nlp.NewNavService(service.NewNavEmbedder(modelProviderService, ""))+ apiNavService.SetNavMergeLLM(service.NewNavMergeLLM(modelProviderService, ""))+ nav.SetNavService(apiNavService)// Install the compiled-wiki search service. It is backed directly by the// document engine: QueryPages filters the tenant-scoped index todiff --git a/internal/service/nav/nav.go b/internal/service/nav/nav.goindex ff1405cf86..c1d77d658f 100644--- a/internal/service/nav/nav.go+++ b/internal/service/nav/nav.go@@ -37,6 +37,14 @@ type NavNode struct {Type string `json:"type"`DocID string `json:"doc_id,omitempty"`HasChildren bool `json:"has_children"`+ // Parent is the tree edge: the parent cluster's name, or "root" for a+ // depth-0 cluster. A search result returns matched leaves TOGETHER with the+ // cluster path above them, so the caller nests the flat list into one tree+ // per root cluster instead of rendering every hit as its own tree.+ Parent string `json:"parent_kwd,omitempty"`+ // Matched marks a keyword-search HIT; the path rows returned alongside it are+ // structure only. Rows outside a search never set it.+ Matched bool `json:"matched,omitempty"`}// Nav row types as stored in the compiled nav index (type_kwd).@@ -69,8 +77,11 @@ type UpsertDocInput struct {// production chat invoker without an import cycle; nil disables LLM behavior and// the implementation falls back to deterministic naming.type NavMergeLLM interface {- // Merge returns a merged cluster description for one or more source texts- // (temperature 0.1, mirroring Python _llm_merge).+ // Merge fuses a new document summary into a cluster's description+ // (temperature 0.1, mirroring Python _llm_merge). texts is+ // [existing description, new document summary]; on failure it must return the+ // existing description (an error also keeps it), so a model hiccup never+ // blanks a stored description.Merge(ctx context.Context, tenantID string, texts []string) (string, error)// CreateSummary returns a short cluster name + summary for a source text// (mirroring Python _llm_create_summary). Return name="" to fall back.@@ -96,17 +107,22 @@ type NavService interface {// returned coverage is trimmed to the scope so no out-of-scope document// surfaces under a cluster that merely overlaps it.Search(ctx context.Context, tenantID, kbID, query string, embd []float32, docScope []string, topK int) ([]NavHit, error)- // ListClusters returns the depth-0 clusters (parent_kwd=root). When- // keywords is non-empty, it returns matching navigation nodes instead.+ // ListClusters returns the depth-0 clusters (parent_kwd=root), or — when+ // keywords is non-empty — the pruned search forest: the matched leaves plus+ // the cluster path above each of them. The total is a cluster count in both+ // modes.ListClusters(ctx context.Context, tenantID, kbID, keywords string, page, pageSize int) ([]NavNode, int64, error)// ListChildren returns the direct children of a cluster (parent_kwd=name),// optionally filtered by keywords.ListChildren(ctx context.Context, tenantID, kbID, name, keywords string, page, pageSize int) ([]NavNode, int64, error)- // SummariesByDocIDs returns the nav_doc summary (preferred readable name, else- // the payload description) keyed by doc_id for the given documents. It mirrors- // Python dataset_api_service._nav_doc_summaries and backs the chunk_agg- // navigation-tree router's document labels. A doc without a nav_doc row is- // absent from the result.+ // SummariesByDocIDs returns the nav_doc description — the document's overall+ // summary, as stored in the row's payload — keyed by doc_id for the given+ // documents, falling back to the row's readable label when the payload+ // carries no description. It mirrors Python+ // dataset_api_service._nav_doc_summaries and backs the chunk_agg+ // navigation-tree router's document labels, which callers render to the model+ // as the document's <summary>. A doc without a nav_doc row is absent from the+ // result.SummariesByDocIDs(ctx context.Context, tenantID, kbID string, docIDs []string) map[string]string}diff --git a/internal/service/nav_merge_llm.go b/internal/service/nav_merge_llm.gonew file mode 100644index 0000000000..afd497db14--- /dev/null+++ b/internal/service/nav_merge_llm.go@@ -0,0 +1,258 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "encoding/json"+ "fmt"+ "strings"++ "ragflow/internal/entity"+ modelModule "ragflow/internal/entity/models"+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// navNamingTemperature is the decoding temperature Python pins for both cluster+// prompts (_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1})).+const navNamingTemperature = 0.1++// navNamingMaxRetries mirrors Python gen_json's max_retry=2 for the JSON naming+// prompt: each retry re-issues the call with the parse error appended, so a+// formatting hiccup costs two calls, not the compiler's default five.+const navNamingMaxRetries = 2++// NavMergeLLM is the production implementation of the nav.NavMergeLLM seam: it+// names dataset-nav clusters and fuses cluster descriptions with the tenant's+// chat model, mirroring Python dataset_nav._llm_create_summary / _llm_merge.+//+// It sits next to NavEmbedder in the service package, not with the seam it+// implements, because it needs ModelProviderService and no inner package may+// import service: service already imports nlp (chat_pipeline, deep_researcher,+// memory) and nav (dataset_artifact_service). The nav package is deliberately a+// dependency-light leaf — the agent tool layer and the nlp implementation both+// depend on it — so only the seam lives there and the model-backed adapters stay+// out here. Moving this type into nav would need the model access reduced to an+// injected interface first.+type NavMergeLLM struct {+ modelSvc *ModelProviderService+ // llmID is the configured chat model ref (a composite+ // "model@instance@provider" name or a tenant model id). Empty resolves the+ // tenant's default chat model, matching the dataset-level deduper's fallback.+ llmID string+}++// NewNavMergeLLM builds the production nav cluster namer/merger.+func NewNavMergeLLM(modelSvc *ModelProviderService, llmID string) *NavMergeLLM {+ return &NavMergeLLM{modelSvc: modelSvc, llmID: llmID}+}++// navNamingPrompt is the cluster-naming prompt (Python _llm_create_summary) plus+// a language constraint. The prompt is written in English, so the model answered+// in English even for a Chinese dataset, and the cluster name it produced (with+// the deterministic hash suffix appended) is what the nav tree displays — an+// English label over Chinese documents. The excerpts' own language is the one+// the tree's readers use, so it wins.+func navNamingPrompt(text string) string {+ return "Given the document excerpts below, produce a short human-readable topic " ++ "name and a concise description of their common topic.\n\n" ++ text + "\n\n" ++ "Write both the name and the description in the same language as the " ++ "excerpts (do not translate them into another language).\n\n" ++ `Return ONLY JSON: {"name": "<2-6 word topic title>", "summary": "<1-3 sentence description>"}`+}++// navMergePrompt fuses an existing cluster description with a new document+// summary (Python _llm_merge). Same language rule, anchored on the NEW text: a+// cluster whose description was written in another language converges back to+// the corpus language as documents are merged in, instead of drifting.+func navMergePrompt(existing, added string) string {+ return "Merge the following two descriptions of the same topic into " ++ "a single concise summary (1-3 sentences):\n\n" ++ "Existing: " + existing + "\n\n" ++ "New: " + added + "\n\n" ++ "Write the merged summary in the same language as the New description " ++ "(do not translate it).\n\n" ++ "Return ONLY the merged text, no commentary."+}++// CreateSummary asks the model for a short topic name plus a 1-3 sentence+// description of the supplied document text (Python _llm_create_summary). An+// empty name or an error makes the caller fall back to the deterministic+// "<summary first line> <hash>" name.+func (l *NavMergeLLM) CreateSummary(ctx context.Context, tenantID, text string) (name, summary string, err error) {+ text = strings.TrimSpace(text)+ if text == "" {+ return "", "", nil+ }+ llmID, err := l.chatModelID(ctx, tenantID)+ if err != nil {+ return "", "", err+ }+ temp := navNamingTemperature+ reply, err := kccommon.GenJSON(ctx, navChatInvoker{modelSvc: l.modelSvc, tenantID: tenantID, llmID: llmID},+ kccommon.ChatRequest{UserPrompt: navNamingPrompt(text), Temperature: &temp}, navNamingMaxRetries)+ if err != nil {+ return "", "", err+ }+ name, summary = navSummaryFromReply(reply, text)+ return name, summary, nil+}++// Merge fuses the existing cluster description with a new document summary into+// one concise summary (Python _llm_merge(chat_mdl, cluster_desc, doc_summary)).+// texts is [existing description, new document summary]; the existing text is+// returned unchanged when there is nothing to merge or the model reply is+// unusable, so the caller keeps the stored description.+func (l *NavMergeLLM) Merge(ctx context.Context, tenantID string, texts []string) (string, error) {+ existing := ""+ if len(texts) > 0 {+ existing = strings.TrimSpace(texts[0])+ }+ added := ""+ if len(texts) > 1 {+ added = strings.TrimSpace(texts[1])+ }+ if added == "" {+ return existing, nil+ }+ llmID, err := l.chatModelID(ctx, tenantID)+ if err != nil {+ return "", err+ }+ temp := navNamingTemperature+ reply, err := navChatText(ctx, l.modelSvc, tenantID, llmID, "", navMergePrompt(existing, added), &temp)+ if err != nil {+ return "", err+ }+ return navMergedFromReply(reply, existing), nil+}++// chatModelID resolves the chat model ref to call: the configured one, else the+// tenant's default chat model.+func (l *NavMergeLLM) chatModelID(ctx context.Context, tenantID string) (string, error) {+ if l.modelSvc == nil {+ return "", fmt.Errorf("datasetnav: model provider service not initialized")+ }+ if ref := strings.TrimSpace(l.llmID); ref != "" {+ return ref, nil+ }+ target, err := l.modelSvc.modelSolver().ResolveDefaultModelConfig(ctx, tenantID, entity.ModelTypeChat)+ if err != nil {+ return "", fmt.Errorf("datasetnav: resolve default chat model for tenant %s: %w", tenantID, err)+ }+ if target == nil || strings.TrimSpace(target.ModelID) == "" {+ return "", fmt.Errorf("datasetnav: no default chat model for tenant %s", tenantID)+ }+ return target.ModelID, nil+}++// navSummaryFromReply mirrors Python _llm_create_summary's response handling:+// name from "name", summary from "summary" then "result", the input text as the+// last resort. An empty name is returned as-is so the caller falls back to its+// deterministic name.+func navSummaryFromReply(reply map[string]any, fallbackSummary string) (name, summary string) {+ summary = navReplyString(reply["summary"])+ if summary == "" {+ summary = navReplyString(reply["result"])+ }+ if summary == "" {+ summary = strings.TrimSpace(fallbackSummary)+ }+ return navReplyString(reply["name"]), summary+}++// navMergedFromReply mirrors Python _llm_merge's reply handling, with one+// deliberate deviation: a plain-text reply IS used. Python feeds the reply+// through gen_json, whose json_repair parse of ordinary prose yields an empty+// string, so its `isinstance(resp, str) and resp.strip()` branch never fires and+// the merge silently keeps the old description — even though the prompt asks for+// "only the merged text". Keeping the reply body is what the prompt intends.+func navMergedFromReply(reply, existing string) string {+ reply = strings.TrimSpace(reply)+ if reply == "" {+ return existing+ }+ if repaired, err := kccommon.RepairJSONText(reply); err == nil {+ var m map[string]any+ if json.Unmarshal([]byte(repaired), &m) == nil {+ if merged := navReplyString(m["merged"]); merged != "" {+ return merged+ }+ if merged := navReplyString(m["result"]); merged != "" {+ return merged+ }+ return existing+ }+ }+ return reply+}++// navReplyString reads a string field out of an LLM JSON reply. Non-string+// values are ignored: the prompts only ever ask for strings, and stringifying a+// nested object would leak a debug repr into a stored description.+func navReplyString(v any) string {+ s, ok := v.(string)+ if !ok {+ return ""+ }+ return strings.TrimSpace(s)+}++// navChatInvoker adapts the model provider service to the knowledge-compiler+// ChatInvoker seam, so the JSON naming prompt reuses kccommon.GenJSON's+// extraction/repair/retry handling (the Go counterpart of Python gen_json).+type navChatInvoker struct {+ modelSvc *ModelProviderService+ tenantID string+ llmID string+}++func (c navChatInvoker) Chat(ctx context.Context, req kccommon.ChatRequest) (*kccommon.ChatResponse, error) {+ content, err := navChatText(ctx, c.modelSvc, c.tenantID, c.llmID, req.SystemPrompt, req.UserPrompt, req.Temperature)+ if err != nil {+ return nil, err+ }+ return &kccommon.ChatResponse{Content: content}, nil+}++// navChatText dispatches one non-streaming chat call. system may be empty (both+// nav prompts are single-turn, matching Python's gen_json("", prompt, ...)).+func navChatText(ctx context.Context, modelSvc *ModelProviderService, tenantID, llmID, system, user string, temperature *float64) (string, error) {+ if modelSvc == nil {+ return "", fmt.Errorf("datasetnav: model provider service not initialized")+ }+ if strings.TrimSpace(llmID) == "" {+ return "", fmt.Errorf("datasetnav: chat model is not resolved")+ }+ config := &modelModule.ChatConfig{}+ if temperature != nil {+ config.Temperature = temperature+ }+ msgs := []modelModule.Message{+ {Role: "system", Content: system},+ {Role: "user", Content: user},+ }+ resp, err := modelSvc.Chat(ctx, tenantID, llmID, msgs, config)+ if err != nil {+ return "", err+ }+ if resp == nil || resp.Answer == nil {+ return "", nil+ }+ return strings.TrimSpace(*resp.Answer), nil+}diff --git a/internal/service/nav_merge_llm_test.go b/internal/service/nav_merge_llm_test.gonew file mode 100644index 0000000000..a983913349--- /dev/null+++ b/internal/service/nav_merge_llm_test.go@@ -0,0 +1,165 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "strings"+ "testing"+)++// TestNavPromptsConstrainLanguage pins the language rule on both cluster prompts.+// The prompts are English, and without the rule the model answered in English+// even for a Chinese dataset: the cluster name it returns (plus the deterministic+// hash suffix) is what the nav tree displays, and its "summary" is the cluster+// description the artifacts page shows.+func TestNavPromptsConstrainLanguage(t *testing.T) {+ naming := navNamingPrompt("赵云的英勇事迹与忠诚品质\n周瑜是东吴大都督……")+ if !strings.Contains(naming, "same language as the excerpts") {+ t.Errorf("naming prompt lost its language constraint: %s", naming)+ }+ // The source text and the JSON contract must survive the extra instruction.+ if !strings.Contains(naming, "赵云的英勇事迹与忠诚品质") {+ t.Error("naming prompt dropped the source text")+ }+ if !strings.Contains(naming, `Return ONLY JSON: {"name": "<2-6 word topic title>"`) {+ t.Error("naming prompt dropped the JSON contract")+ }++ merging := navMergePrompt("Three Kingdoms Historical Figures", "赵云的英勇事迹与忠诚品质")+ if !strings.Contains(merging, "same language as the New description") {+ t.Errorf("merge prompt lost its language constraint: %s", merging)+ }+ if !strings.Contains(merging, "Existing: Three Kingdoms Historical Figures") ||+ !strings.Contains(merging, "New: 赵云的英勇事迹与忠诚品质") {+ t.Error("merge prompt dropped one of the two descriptions")+ }+ if !strings.Contains(merging, "Return ONLY the merged text, no commentary.") {+ t.Error("merge prompt dropped its output contract")+ }+}++// TestNavSummaryFromReply pins the naming reply contract (Python+// _llm_create_summary): "name" is the topic title, "summary" falls back to+// "result" and then to the source text. An empty name is surfaced as-is so the+// caller keeps its deterministic "<summary first line> <hash>" name.+func TestNavSummaryFromReply(t *testing.T) {+ tests := []struct {+ name string+ reply map[string]any+ fallback string+ wantName string+ wantSummary string+ }{+ {+ name: "name and summary",+ reply: map[string]any{"name": "三国名将事迹", "summary": "三国名将的事迹概览。"},+ fallback: "source summary",+ wantName: "三国名将事迹",+ wantSummary: "三国名将的事迹概览。",+ },+ {+ name: "summary key missing falls back to result",+ reply: map[string]any{"name": " topic ", "result": "merged text"},+ fallback: "source summary",+ wantName: "topic",+ wantSummary: "merged text",+ },+ {+ name: "no usable fields keeps the source text",+ reply: map[string]any{},+ fallback: "source summary",+ wantName: "",+ wantSummary: "source summary",+ },+ {+ name: "name missing leaves the caller to fall back",+ reply: map[string]any{"summary": "only a summary"},+ fallback: "source summary",+ wantName: "",+ wantSummary: "only a summary",+ },+ {+ name: "non-string values are ignored, not stringified",+ reply: map[string]any{"name": 42, "summary": map[string]any{"x": 1}},+ fallback: "source summary",+ wantName: "",+ wantSummary: "source summary",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ gotName, gotSummary := navSummaryFromReply(tt.reply, tt.fallback)+ if gotName != tt.wantName || gotSummary != tt.wantSummary {+ t.Errorf("navSummaryFromReply() = (%q, %q), want (%q, %q)",+ gotName, gotSummary, tt.wantName, tt.wantSummary)+ }+ })+ }+}++// TestNavMergedFromReply pins the merge reply contract (Python _llm_merge), with+// the deliberate deviation documented on the function: a plain-text reply is+// used (the prompt asks for "only the merged text"), while a JSON reply prefers+// "merged" then "result" and an unusable reply keeps the existing description.+func TestNavMergedFromReply(t *testing.T) {+ tests := []struct {+ name string+ reply string+ existing string+ want string+ }{+ {+ name: "plain text reply is the merged summary",+ reply: "三国名将与曹操的对比,两人才能各具特色。",+ existing: "old description",+ want: "三国名将与曹操的对比,两人才能各具特色。",+ },+ {+ name: "merged key wins",+ reply: `{"merged": "fused description"}`,+ existing: "old description",+ want: "fused description",+ },+ {+ name: "result key is accepted",+ reply: `{"result": "fused via result"}`,+ existing: "old description",+ want: "fused via result",+ },+ {+ name: "json without the expected keys keeps the existing text",+ reply: `{"unexpected": "value"}`,+ existing: "old description",+ want: "old description",+ },+ {+ name: "empty reply keeps the existing text",+ reply: "",+ existing: "old description",+ want: "old description",+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := navMergedFromReply(tt.reply, tt.existing); got != tt.want {+ t.Errorf("navMergedFromReply(%q) = %q, want %q", tt.reply, got, tt.want)+ }+ })+ }+}diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.goindex 49190bc78c..bfe0ad8c53 100644--- a/internal/service/nlp/datasetnav.go+++ b/internal/service/nlp/datasetnav.go@@ -23,10 +23,13 @@ import ("sort""strings"+ "ragflow/internal/common""ragflow/internal/engine""ragflow/internal/engine/types""ragflow/internal/service/nav""ragflow/internal/tokenizer"++ "go.uber.org/zap")// Dataset-nav constants. These mirror Python's dataset_nav.py: nav rows live in@@ -47,8 +50,22 @@ const (// the children-search cost when the cluster is plausibly overfull.navMaxFanout = 64 // max direct children before rebalancenavMaxDocsPerCluster = 50 // max docs per leaf cluster before split++ // navTreeReadLimit bounds the single read a search uses to resolve the+ // ancestor path of its hits. A nav tree is small by construction (fanout,+ // docs-per-cluster and depth are all capped), so this is a guard against a+ // pathological dataset rather than a paging budget.+ navTreeReadLimit = 10000)+// navNodeSelectFields are the columns every nav tree read needs: the label+// (title_kwd), the Python-era name column, the parent edge, the payload+// (description), and the leaf's document id.+var navNodeSelectFields = []string{+ "name", "title_kwd", "parent_kwd", "depth_int",+ "content_with_weight", "doc_count_int", "type_kwd", "doc_id",+}+// NavEmbedder embeds text. tenantID lets a production implementation resolve// the tenant's embedding model. Kept as an interface so tests inject a stub.type NavEmbedder interface {@@ -86,10 +103,10 @@ func NewNavService(embed NavEmbedder) *NavService {return &NavService{embed: embed}}-// SetNavMergeLLM installs the optional LLM used for cluster-description merging-// and summary naming (mirroring Python dataset_nav._llm_merge/_llm_create_summary).-// When nil (the default), NavService falls back to deterministic naming and-// concatenated descriptions.+// SetNavMergeLLM installs the optional LLM used for cluster naming and+// description merging (mirroring Python dataset_nav._llm_create_summary /+// _llm_merge). When nil (the default), clusters keep the deterministic+// "<summary first line> <hash>" name and their existing description.func (s *NavService) SetNavMergeLLM(llm nav.NavMergeLLM) {s.llm = llm}@@ -149,24 +166,27 @@ func (s *NavService) navSearch(ctx context.Context, tenantID, kbID string, filtereturn res.Chunks, res.Total, nil}-// ListClusters returns the depth-0 clusters (parent_kwd=root). When keywords-// is non-empty, it searches all navigation rows by their indexed summary and-// returns matching nodes, mirroring the Python endpoint's search mode.+// ListClusters returns the depth-0 clusters (parent_kwd=root). When keywords is+// non-empty it searches the dataset's documents and returns the pruned forest+// around the hits (see searchNavForest) instead: one tree per root cluster,+// holding only the cluster path down to each matched nav_doc leaf.+//+// The total is a cluster count in both modes: the depth-0 clusters when browsing,+// the clusters holding a match when searching.func (s *NavService) ListClusters(ctx context.Context, tenantID, kbID, keywords string, page, pageSize int) ([]nav.NavNode, int64, error) {if pageSize <= 0 {pageSize = 100}+ if strings.TrimSpace(keywords) != "" {+ return s.searchNavForest(ctx, tenantID, kbID, keywords, pageSize)+ }offset := page * pageSizefilter := navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"},"parent_kwd": []string{navRootParent},})- if strings.TrimSpace(keywords) != "" {- filter = navFilter(nil)- }chunks, total, err := s.navSearch(ctx, tenantID, kbID, filter,- []string{"name", "title_kwd", "content_with_weight", "doc_count_int", "type_kwd", "doc_id"}, offset, pageSize,- navKeywordExpressions(keywords))+ navNodeSelectFields, offset, pageSize, navKeywordExpressions(keywords))if err != nil {return nil, 0, err}@@ -186,7 +206,7 @@ func (s *NavService) ListChildren(ctx context.Context, tenantID, kbID, name, keyoffset := page * pageSizechunks, total, err := s.navSearch(ctx, tenantID, kbID,navFilter(map[string]interface{}{"parent_kwd": []string{name}}),- []string{"name", "title_kwd", "content_with_weight", "doc_count_int", "type_kwd", "doc_id"}, offset, pageSize,+ navNodeSelectFields, offset, pageSize,navKeywordExpressions(keywords))if err != nil {return nil, 0, err@@ -203,6 +223,135 @@ func (s *NavService) ListChildren(ctx context.Context, tenantID, kbID, name, keyreturn nodes, total, nil}+// searchNavForest resolves a keyword search into the pruned forest the nav tree+// renders: the matched nav_doc leaves PLUS the cluster path from each match up to+// its root cluster, so every root cluster stays ONE tree holding only the+// branches that lead to a match. The rows carry parent_kwd (the tree edge) and+// matched (hits only), so the caller nests the flat list without another request.+//+// The total counts the clusters the hits landed in (see searchNavClusterCount):+// the same quantity the header shows while browsing. Matching is document-level+// on purpose (Python's tree search routes to documents too): a cluster appears+// only as the path above a matched leaf, so a cluster's own summary text alone+// does not surface a tree.+func (s *NavService) searchNavForest(ctx context.Context, tenantID, kbID, keywords string, limit int) ([]nav.NavNode, int64, error) {+ matches, _, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{"type_kwd": []string{nav.TypeNavDoc}}),+ navNodeSelectFields, 0, limit, navKeywordExpressions(keywords))+ if err != nil {+ return nil, 0, err+ }+ total := searchNavClusterCount(matches)+ if len(matches) == 0 {+ return []nav.NavNode{}, total, nil+ }+ // One read of the dataset's nav rows resolves every ancestor chain at once.+ // A nav tree is small by construction (fanout, docs-per-cluster and depth are+ // all capped), so this beats a parent-by-parent walk of navMaxDepth queries.+ rows, _, err := s.navSearch(ctx, tenantID, kbID, navFilter(nil), navNodeSelectFields, 0, navTreeReadLimit, nil)+ if err != nil {+ // The path lookup failed; the hits are still a valid — if flat — answer.+ common.Warn("datasetnav: nav path lookup for the search forest failed",+ zap.String("kb_id", kbID), zap.Error(err))+ return s.navNodesFromRows(matches), total, nil+ }+ clusters := make(map[string]map[string]interface{}, len(rows))+ for _, r := range rows {+ if firstStringValue(r["type_kwd"]) != nav.TypeNavCluster {+ continue+ }+ if key := navClusterRowKey(r); key != "" {+ clusters[key] = r+ }+ }++ kept := make(map[string]map[string]interface{}, len(matches))+ hit := make(map[string]bool, len(matches))+ for _, r := range matches {+ docID := firstStringValue(r["doc_id"])+ if docID == "" {+ continue+ }+ key := navDocRowKey(docID)+ kept[key] = r+ hit[key] = true+ // Walk up to (but excluding) the root sentinel. navMaxDepth bounds the+ // walk so a corrupt parent cycle can never spin.+ parent := firstStringValue(r["parent_kwd"])+ for depth := 0; parent != "" && parent != navRootParent && depth < navMaxDepth; depth++ {+ cluster, ok := clusters[parent]+ if !ok {+ break // orphaned edge: the path above it is unknown+ }+ kept[navClusterKey(parent)] = cluster+ parent = firstStringValue(cluster["parent_kwd"])+ }+ }+ // Deterministic order (shallowest first, then by key): the caller re-nests+ // anyway, but a stable order keeps the response diffable and testable.+ keys := make([]string, 0, len(kept))+ for key := range kept {+ keys = append(keys, key)+ }+ sort.Slice(keys, func(i, j int) bool {+ if di, dj := intValue(kept[keys[i]]["depth_int"]), intValue(kept[keys[j]]["depth_int"]); di != dj {+ return di < dj+ }+ return keys[i] < keys[j]+ })+ nodes := make([]nav.NavNode, 0, len(keys))+ for _, key := range keys {+ node := s.nodeFromRow(kept[key], "doc")+ node.Matched = hit[key]+ nodes = append(nodes, node)+ }+ return nodes, total, nil+}++// searchNavClusterCount returns the number of clusters a search's hits landed in+// — the header's number in search mode, like the depth-0 cluster count it shows+// while browsing. A hit leaf sits in exactly one cluster (its parent_kwd), so+// distinct parents count the clusters touched: two hits in one cluster count+// once, and an ancestor shared by hits in different sub-clusters is a path row,+// not a second cluster.+func searchNavClusterCount(matches []map[string]interface{}) int64 {+ seen := make(map[string]bool, len(matches))+ for _, r := range matches {+ // A leaf with no cluster edge (no writer produces one) has no cluster to+ // count.+ if key := firstStringValue(r["parent_kwd"]); key != "" && key != navRootParent {+ seen[key] = true+ }+ }+ return int64(len(seen))+}++// navNodesFromRows maps engine rows to NavNodes, all marked as hits (the degrade+// path of searchNavForest, where no path rows were resolved).+func (s *NavService) navNodesFromRows(rows []map[string]interface{}) []nav.NavNode {+ nodes := make([]nav.NavNode, 0, len(rows))+ for _, r := range rows {+ node := s.nodeFromRow(r, "doc")+ node.Matched = true+ nodes = append(nodes, node)+ }+ return nodes+}++// navClusterRowKey is the key a cluster's children reference through parent_kwd:+// Python's writer stores it in `name`, Go's in `title_kwd`.+func navClusterRowKey(row map[string]interface{}) string {+ if name := firstStringValue(row["name"]); name != "" {+ return name+ }+ return firstStringValue(row["title_kwd"])+}++// navDocRowKey / navClusterKey namespace the two row identities so a cluster and+// a document can never collide in the search forest's keyed sets.+func navDocRowKey(docID string) string { return "doc:" + docID }+func navClusterKey(name string) string { return "cluster:" + name }+// navKeywordExpressions builds the lexical search expression used by the REST// navigation endpoints. These are the same indexed summary fields used by the// Python navigation search, with a stronger weight on the main content field.@@ -222,10 +371,13 @@ func navKeywordExpressions(keywords string) []interface{} {}// nodeFromRow converts an engine row into a NavNode.+//+// The label is title_kwd — the summary's first line: one column serves both the+// tree and the model-facing readers (NavHit.Name), so no separate display column+// is written. An explicit readable `name` column (Python's writer stores one)+// wins when present.func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string) nav.NavNode {name := firstStringValue(row["title_kwd"])- // Prefer an explicit readable "name" column (Python rows carry one); fall- // back to title_kwd.if n := firstStringValue(row["name"]); n != "" {name = n}@@ -243,6 +395,7 @@ func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType stringDocCount: intValue(row["doc_count_int"]),Type: fallbackType,DocID: firstStringValue(row["doc_id"]),+ Parent: firstStringValue(row["parent_kwd"]),}if t := firstStringValue(row["type_kwd"]); t != "" {if t == "nav_cluster" {@@ -476,11 +629,15 @@ func navIntersectScope(docIDs []string, allowed map[string]struct{}) []string {return out}-// SummariesByDocIDs returns the nav_doc summary keyed by doc_id for the given-// documents. It mirrors Python dataset_api_service._nav_doc_summaries: it reads-// the nav_doc rows (compile_kwd=dataset_nav, type_kwd=nav_doc) for the doc_ids-// and yields, per doc, the readable nav name when present, else the payload-// description. Documents with no nav_doc row are absent from the result.+// SummariesByDocIDs returns the nav_doc description — the document's overall+// summary — keyed by doc_id for the given documents. It mirrors Python+// dataset_api_service._nav_doc_summaries: it reads the nav_doc rows+// (compile_kwd=dataset_nav, type_kwd=nav_doc) for the doc_ids and yields, per+// doc, the payload description. The description wins over the row's label on+// purpose: callers render this value to the model as the routed document's+// <summary>, and title_kwd holds only the summary's first line (length-capped).+// A row whose payload carries no description falls back to its readable label; a+// document with no nav_doc row is absent from the result.func (s *NavService) SummariesByDocIDs(ctx context.Context, tenantID, kbID string, docIDs []string) map[string]string {if len(docIDs) == 0 {return map[string]string{}@@ -501,24 +658,25 @@ func (s *NavService) SummariesByDocIDs(ctx context.Context, tenantID, kbID strinif docID == "" {continue}- // Prefer an explicit readable name, else the payload description, matching- // nodeFromRow's label rules (a raw 32-hex id is not a human label).- name := firstStringValue(c["name"])- if name == "" {- name = firstStringValue(c["title_kwd"])- }- if graphIsRawID(name) {- name = ""+ summary := ""+ if payload, ok := c["content_with_weight"].(string); ok {+ var m map[string]interface{}+ if json.Unmarshal([]byte(payload), &m) == nil {+ if d, ok := m["description"].(string); ok {+ summary = strings.TrimSpace(d)+ }+ }}- summary := nameif summary == "" {- if payload, ok := c["content_with_weight"].(string); ok {- var m map[string]interface{}- if json.Unmarshal([]byte(payload), &m) == nil {- if d, ok := m["description"].(string); ok {- summary = strings.TrimSpace(d)- }- }+ // No description in the payload: fall back to the stored label+ // (readable name, else title_kwd), rejecting raw ids — a 32-hex id or+ // "cluster_<hash>" is not a human label.+ summary = firstStringValue(c["name"])+ if summary == "" {+ summary = firstStringValue(c["title_kwd"])+ }+ if graphIsRawID(summary) {+ summary = ""}}if summary != "" {@@ -568,18 +726,35 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"type_kwd": []string{nav.TypeNavDoc},"doc_id": []string{in.DocID},}),- []string{"content_with_weight", "content_ltks", "content_sm_ltks"}, 0, 1, nil)+ []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd"}, 0, 1, nil)if err != nil {return err}if len(existing) > 0 {+ // The stored label can drift without the summary changing, so a recompile+ // refreshes it in place instead of leaving a stale row: a row written by an+ // older label rule (title_kwd holding the file name) goes back to the+ // summary's first line, or the model-facing readers would keep seeing a+ // file name. The tokenized text is (re)built when it is missing.+ title := navDocTitle(in.Summary)+ titleStale := firstStringValue(existing[0]["title_kwd"]) != title+ ltksStale := firstStringValue(existing[0]["content_ltks"]) == "" ||+ firstStringValue(existing[0]["content_sm_ltks"]) == ""if payload, ok := existing[0]["content_with_weight"].(string); ok {var m map[string]interface{}if err := json.Unmarshal([]byte(payload), &m); err == nil {if d, _ := m["description"].(string); d == in.Summary {- if firstStringValue(existing[0]["content_ltks"]) != "" && firstStringValue(existing[0]["content_sm_ltks"]) != "" {+ if !titleStale && !ltksStale {return nil // unchanged}+ updates := map[string]interface{}{}+ if titleStale {+ updates["title_kwd"] = title+ }+ if ltksStale {+ updates["content_ltks"] = contentLtks+ updates["content_sm_ltks"] = contentSmLtks+ }return de.UpdateChunks(ctx,map[string]interface{}{"compile_kwd": []string{navCompileKwd},@@ -587,10 +762,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"doc_id": []string{in.DocID},"kb_id": in.KbID,},- map[string]interface{}{- "content_ltks": contentLtks,- "content_sm_ltks": contentSmLtks,- }, s.navIndexName(in.TenantID), in.KbID)+ updates, s.navIndexName(in.TenantID), in.KbID)}}}@@ -619,6 +791,14 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) errorif bestName != "" && sim >= navMergeThreshold {parent := bestName+ // Fuse this document's summary into the cluster description and re-embed+ // the cluster with the merged text (Python upsert_dataset_nav_doc's merge+ // branch), so the cluster keeps describing every document it holds and the+ // next placement scores against the merged topic instead of the first+ // document's summary alone.+ if err := s.mergeClusterDescription(ctx, de, in.TenantID, in.KbID, bestName, in.Summary); err != nil {+ return err+ }if err := s.appendDocToCluster(ctx, de, in.TenantID, in.KbID, bestName, in.DocID,fmt.Sprintf("q_%d_vec", len(vec))); err != nil {return err@@ -632,10 +812,12 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"compile_kwd": navCompileKwd,"available_int": 0,"type_kwd": "nav_doc",- // The nav_doc's display name is a readable title derived from the summary- // (Python _clean_title/_fallback_title) — NOT the raw doc id, which is- // meaningless in the UI. The doc_id is still stored for lookups.- "title_kwd": cleanTitle(fallbackTitle(in.Summary)),+ // The nav_doc's label is a readable title derived from the summary+ // (Python _clean_title/_fallback_title) — never the raw doc id,+ // which is meaningless in the UI. The doc_id is still stored for+ // lookups; NavHit.Name reads this same column, so the tree and the+ // model-facing readers stay on one label.+ "title_kwd": navDocTitle(in.Summary),"parent_kwd": parent,// The nav_doc sits one level below its (possibly nested) parent// cluster, so its depth is parentDepth+1 — not a hard-coded 1.@@ -651,15 +833,22 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) errorreturn err}- // A similar-but-not-mergeable cluster creates a sibling sub-cluster (Python- // _MIN_SIM=0.50); otherwise a fresh root cluster. This keeps the nav tree- // from degrading into one root per document.+ // A similar-but-not-mergeable document opens its own cluster (Python+ // _MIN_SIM=0.50). Where it lands follows the best match's depth:+ //+ // - a ROOT-level best match opens a ROOT-level cluster, beside it. That is+ // the case a fresh dataset always reaches, and it is what Python's branch+ // does: it attaches the new cluster to `best_parent`, which for a root+ // best match is the root sentinel. Hiding the new topic one level down+ // instead (what this branch used to do) meant a second topic never showed+ // up in the tree's top level.+ // - a best match that is itself nested keeps the child semantics: the new+ // cluster sits one level deeper, which is what keeps a large tree from+ // flattening into one root per topic.parent := navRootParentdepth := 0- if bestName != "" && sim >= navMinSim {+ if bestName != "" && sim >= navMinSim && bestDepth > 0 {parent = bestName- // A sibling of the (possibly nested) best cluster is one level deeper- // than it, so depth = parentDepth+1 — not a hard-coded 1.depth = bestDepth + 1}// New cluster: when an LLM is configured, generate a short name + summary@@ -707,7 +896,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"compile_kwd": navCompileKwd,"available_int": 0,"type_kwd": "nav_doc",- "title_kwd": cleanTitle(fallbackTitle(in.Summary)),+ "title_kwd": navDocTitle(in.Summary),"parent_kwd": name,"depth_int": depth + 1,"doc_id": in.DocID,@@ -788,9 +977,10 @@ func (s *NavService) cleanupEmptyCluster(ctx context.Context, tenantID, kbID, cl// findBestCluster finds the best-matching cluster via level-by-level descent// (mirroring Python _find_best_cluster). It KNNs the current level's clusters-// and, when the best match is >= recurse threshold, descends into that cluster's-// children. Returns the best cluster name, similarity, and its depth (0 = root)-// so callers can assign consistent child depth_int values.+// and, while the match is >= recurse threshold, descends into that cluster's+// children. Returns the DEEPEST cluster reached — the strongest match is not+// kept, exactly like Python — plus its similarity and depth (0 = root), so the+// caller can assign consistent child depth_int values.func (s *NavService) findBestCluster(ctx context.Context, tenantID, kbID string, vec []float32) (string, float64, int, error) {f64 := f32ToF64Slice(vec)parent := navRootParent@@ -821,10 +1011,14 @@ func (s *NavService) findBestCluster(ctx context.Context, tenantID, kbID string,}name := firstStringValue(chunks[0]["title_kwd"])sim := rowScore(chunks[0])- // Keep the STRONGEST match seen so far across all levels, so a strong- // ancestor is never displaced by a weaker descendant. Record its depth- // so the caller can set consistent child depth_int values.- if sim > bestSim {+ // Adopt-and-overwrite, mirroring Python _find_best_cluster: the result is+ // the DEEPEST cluster reached while the descent keeps clearing+ // navRecurse, not the strongest one seen. A strong ancestor therefore+ // does NOT shadow a weaker-but-recurseable descendant — the placement+ // then decides on the descendant's own similarity. The root (level 0) is+ // adopted unconditionally, exactly as Python seeds its `best` with the+ // top-1 root; a child is adopted only when it still clears the threshold.+ if level == 0 || sim >= navRecurse {bestName, bestSim, bestDepth = name, sim, level}// Descend only while the current match is strong enough that a deeper@@ -1105,16 +1299,89 @@ func clusterDepth(clusterChunks []map[string]interface{}) int {return intValue(clusterChunks[0]["depth_int"])}-// llmMergeDescription merges a set of source descriptions into one, using the-// optional LLM when available (temperature 0.1, mirroring Python _llm_merge);-// without an LLM it concatenates them deterministically.+// mergeClusterDescription fuses one document's summary into an existing+// cluster's description and re-embeds the cluster with the merged text+// (mirroring Python upsert_dataset_nav_doc's merge branch: _llm_merge, then a+// re-embed of the changed description). Nothing is written when there is no LLM+// — Python's _llm_merge returns the existing text when chat_mdl is nil — or when+// the model reply leaves the description unchanged, so a no-LLM deployment keeps+// its previous behaviour.+func (s *NavService) mergeClusterDescription(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docSummary string) error {+ docSummary = strings.TrimSpace(docSummary)+ if docSummary == "" {+ return nil+ }+ rows, _, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),+ []string{"content_with_weight"}, 0, 1, nil)+ if err != nil {+ return err+ }+ if len(rows) == 0 {+ return nil+ }+ payload := map[string]interface{}{}+ if raw, ok := rows[0]["content_with_weight"].(string); ok {+ _ = json.Unmarshal([]byte(raw), &payload)+ }+ oldDesc := firstStringValue(payload["description"])+ if oldDesc == "" {+ // Nothing to fuse with: the row carries no description to preserve.+ return nil+ }+ merged := strings.TrimSpace(s.llmMergeDescription(ctx, tenantID, []string{oldDesc, docSummary}))+ if merged == "" || merged == oldDesc {+ return nil+ }+ // Mutate the parsed payload so any extra keys a writer stored there (Python+ // carries keywords/entities/graph_content) survive the rewrite.+ payload["description"] = merged+ updates := map[string]interface{}{"content_with_weight": payloadJSONNav(payload)}+ // The description is part of the cluster's keyword index, so its tokens are+ // refreshed alongside it.+ if ltks, err := tokenizer.Tokenize(merged); err == nil && ltks != "" {+ updates["content_ltks"] = ltks+ if smLtks, err := tokenizer.FineGrainedTokenize(ltks); err == nil && smLtks != "" {+ updates["content_sm_ltks"] = smLtks+ }+ }+ if s.embed != nil {+ if vecs, err := s.embed.Encode(ctx, tenantID, []string{merged}); err == nil && len(vecs) > 0 && len(vecs[0]) > 0 {+ updates[fmt.Sprintf("q_%d_vec", len(vecs[0]))] = f32ToF64Slice(vecs[0])+ }+ }+ return de.UpdateChunks(ctx,+ map[string]interface{}{+ "compile_kwd": []string{navCompileKwd},+ "type_kwd": []string{"nav_cluster"},+ "title_kwd": []string{clusterName},+ "kb_id": kbID,+ }, updates, s.navIndexName(tenantID), kbID)+}++// llmMergeDescription merges a set of source descriptions into one via the+// optional LLM (temperature 0.1, mirroring Python _llm_merge). texts is+// [existing description, new document summary]. Without an LLM — or when the+// model reply is unusable — the existing description is returned unchanged,+// mirroring Python's fallback (it keeps the old summary rather than+// concatenating).func (s *NavService) llmMergeDescription(ctx context.Context, tenantID string, texts []string) string {+ existing := ""+ if len(texts) > 0 {+ existing = texts[0]+ }if s.llm != nil && len(texts) > 0 {- if merged, err := s.llm.Merge(ctx, tenantID, texts); err == nil && strings.TrimSpace(merged) != "" {+ merged, err := s.llm.Merge(ctx, tenantID, texts)+ if err != nil {+ // Keep the stored description; the warning is the only signal that the+ // configured chat model could not be used (Python logs the same way).+ common.Warn("datasetnav: LLM cluster-description merge failed, keeping the existing description",+ zap.String("tenant_id", tenantID), zap.Error(err))+ } else if strings.TrimSpace(merged) != "" {return merged}}- return strings.Join(texts, "\n")+ return existing}// cleanTitle normalizes a raw title/summary into a one-line, length-capped@@ -1142,6 +1409,16 @@ func fallbackTitle(summary string) string {return line}+// navDocTitle is the nav_doc leaf's label, stored in title_kwd: the summary's+// first line (Python _clean_title/_fallback_title). One column serves both+// readers — nodeFromRow renders it in the tree, and NavHit.Name carries it to+// the model as the routed document's <summary> (tool_navigation.go) — so it must+// stay a CONTENT label; a file name here would tell the model nothing about the+// document.+func navDocTitle(summary string) string {+ return cleanTitle(fallbackTitle(summary))+}+// readableClusterName returns a readable yet unique nav-cluster key of the form// "<title> <8-hex>" (mirroring Python dataset_nav._readable_cluster_name): the// title keeps the node name human-readable, the short hash of the seed keeps the@@ -1154,14 +1431,24 @@ func readableClusterName(title, seed string) string {return t + " " + shortHash8(seed)}-// llmCreateSummary builds a short cluster name + summary for a source text via-// the optional LLM (mirroring Python _llm_create_summary); without an LLM it-// falls back to a readable name derived from the summary's first words plus a-// short hash, so nav clusters are human-readable instead of "cluster_<hash>".+// llmCreateSummary builds a cluster's name + summary for a source text. The+// name is always run through readableClusterName, so it keeps the " <8-hex>"+// suffix that makes it unique per KB and addressable as the cluster key+// (parent_kwd / row id) — Python does the same in its callers+// (_readable_cluster_name(new_title, summary)). With an LLM the title part is+// the model's topic name; without one it is the summary's first line, so nav+// clusters stay human-readable instead of "cluster_<hash>".func (s *NavService) llmCreateSummary(ctx context.Context, tenantID, text string) (name, summary string) {if s.llm != nil {- if n, sm, err := s.llm.CreateSummary(ctx, tenantID, text); err == nil && n != "" {- return n, sm+ n, sm, err := s.llm.CreateSummary(ctx, tenantID, text)+ switch {+ case err != nil:+ // Fall back to the deterministic name; the warning is the only signal+ // that the configured chat model could not be used.+ common.Warn("datasetnav: LLM cluster naming failed, using the deterministic name",+ zap.String("tenant_id", tenantID), zap.Error(err))+ case n != "":+ return readableClusterName(n, text), sm}}return readableClusterName(fallbackTitle(text), text), textdiff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.goindex 753aad9d64..5a70733828 100644--- a/internal/service/nlp/datasetnav_test.go+++ b/internal/service/nlp/datasetnav_test.go@@ -69,10 +69,15 @@ func (m *memNavEngine) DeleteChunks(_ context.Context, cond map[string]interfacefunc (m *memNavEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {var matched []map[string]interface{}- hasDense := len(req.MatchExprs) > 0+ // This engine implements the DENSE leg only. A lexical match expression is+ // deliberately a no-op rather than a dense one: treating a text expression+ // as dense would look for a q_0_vec column and silently drop every row, so a+ // keyword search would look broken in tests.+ hasDense := falsevar queryVec []float64- if hasDense {+ if len(req.MatchExprs) > 0 {if de, ok := req.MatchExprs[0].(*types.MatchDenseExpr); ok {+ hasDense = truequeryVec = de.EmbeddingData}}@@ -365,6 +370,210 @@ func TestNavNode_JSONShape_SnakeCase(t *testing.T) {t.Errorf("NavNode JSON leaked PascalCase key %q; got %s", bad, b)}}+ // A search hit carries the tree edge and the hit flag. Both are optional:+ // the ordinary tree reads (root clusters, children) never set them, so they+ // must stay out of the payload there.+ hit := nav.NavNode{Name: "doc.pdf", Type: "doc", Parent: "topic 8738e200", Matched: true}+ hb, err := json.Marshal(hit)+ if err != nil {+ t.Fatal(err)+ }+ var hm map[string]interface{}+ if err := json.Unmarshal(hb, &hm); err != nil {+ t.Fatal(err)+ }+ if hm["parent_kwd"] != "topic 8738e200" || hm["matched"] != true {+ t.Errorf("search hit JSON = %s, want parent_kwd + matched", hb)+ }+ if _, ok := m["matched"]; ok {+ t.Errorf("a non-hit NavNode must not carry matched; got %s", b)+ }+}++// TestNavService_ListClusters_SearchReturnsPrunedForest pins the search shape:+// the hits come back TOGETHER with the cluster path above them, so the caller+// nests the flat list into one tree per root cluster instead of rendering every+// hit — and every matched subtree — as its own tree.+func TestNavService_ListClusters_SearchReturnsPrunedForest(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ const (+ rootCluster = "三国 11111111"+ subCluster = "名将 22222222"+ )+ rows := []map[string]interface{}{+ {+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavCluster,+ "title_kwd": rootCluster, "parent_kwd": navRootParent, "depth_int": 0,+ "doc_count_int": 2, "doc_ids_kwd": []string{"d1", "d2"},+ "content_with_weight": `{"type":"nav_cluster","description":"三国主题"}`,+ },+ {+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavCluster,+ "title_kwd": subCluster, "parent_kwd": rootCluster, "depth_int": 1,+ "doc_count_int": 1, "doc_ids_kwd": []string{"d1"},+ "content_with_weight": `{"type":"nav_cluster","description":"名将主题"}`,+ },+ {+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,+ "doc_id": "d1", "parent_kwd": subCluster, "depth_int": 2,+ "title_kwd": "三国名将事迹概要", "doc_count_int": 1,+ "content_with_weight": `{"type":"nav_doc","description":"三国名将事迹概要\n正文"}`,+ },+ // A second leaf in subCluster: two hits in ONE cluster must count once.+ {+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,+ "doc_id": "d4", "parent_kwd": subCluster, "depth_int": 2,+ "title_kwd": "名将功过评析", "doc_count_int": 1,+ "content_with_weight": `{"type":"nav_doc","description":"名将功过评析"}`,+ },+ {+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,+ "doc_id": "d2", "parent_kwd": rootCluster, "depth_int": 1,+ "title_kwd": "其他概要", "doc_count_int": 1,+ "content_with_weight": `{"type":"nav_doc","description":"其他概要"}`,+ },+ // An orphan whose parent cluster is gone: the hit is still returned (as+ // its own root) rather than dropped.+ {+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,+ "doc_id": "d3", "parent_kwd": "missing-cluster", "depth_int": 1,+ "title_kwd": "孤儿概要", "doc_count_int": 1,+ "content_with_weight": `{"type":"nav_doc","description":"孤儿概要"}`,+ },+ }+ if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1"); err != nil {+ t.Fatal(err)+ }++ nodes, total, err := ns.ListClusters(t.Context(), "t1", "kb1", "任意关键词", 0, 100)+ if err != nil {+ t.Fatal(err)+ }+ // The test engine ignores the lexical expression, so all four nav_doc rows+ // are hits. total is the CLUSTER count — the same quantity the nav tree+ // header shows while browsing — not the hit count: d1 and d4 share+ // subCluster (one cluster, two hits), d2 sits directly in rootCluster, and+ // d3's cluster edge dangles. The path rows above the hits are not counted.+ if total != 3 {+ t.Errorf("search total = %d, want 3 (the clusters the hits landed in, not the 4 hits)", total)+ }+ byID := make(map[string]nav.NavNode, len(nodes))+ for _, n := range nodes {+ if n.Type == "doc" {+ byID[n.DocID] = n+ }+ }+ if len(byID) != 4 {+ t.Fatalf("search forest returned %d doc nodes, want 4: %+v", len(byID), nodes)+ }+ // The cluster path above d1 (sub-cluster then root) must be present so the+ // tree stays ONE tree, and it must not be flagged as a hit.+ var gotRoot, gotSub bool+ for _, n := range nodes {+ switch n.Name {+ case rootCluster:+ gotRoot = true+ if n.Matched {+ t.Errorf("path cluster %q marked as a hit", n.Name)+ }+ if n.Parent != navRootParent {+ t.Errorf("root cluster parent = %q, want %q", n.Parent, navRootParent)+ }+ case subCluster:+ gotSub = true+ if n.Matched {+ t.Errorf("path cluster %q marked as a hit", n.Name)+ }+ if n.Parent != rootCluster {+ t.Errorf("sub cluster parent = %q, want %q", n.Parent, rootCluster)+ }+ }+ }+ if !gotRoot || !gotSub {+ t.Fatalf("search forest is missing the path above d1 (root=%v sub=%v): %+v", gotRoot, gotSub, nodes)+ }+ // The leaf keeps both the tree edge and its label (title_kwd, the summary's+ // first line — the same column the model-facing readers use).+ if got := byID["d1"]; got.Parent != subCluster || !got.Matched || got.Name != "三国名将事迹概要" {+ t.Errorf("hit d1 = %+v, want parent=%q matched=true name=三国名将事迹概要", got, subCluster)+ }+ if got := byID["d4"]; got.Parent != subCluster || !got.Matched {+ t.Errorf("hit d4 = %+v, want parent=%q matched=true", got, subCluster)+ }+ if got := byID["d2"]; got.Parent != rootCluster || !got.Matched {+ t.Errorf("hit d2 = %+v, want parent=%q matched=true", got, rootCluster)+ }+ if got := byID["d3"]; got.Parent != "missing-cluster" || !got.Matched {+ t.Errorf("orphan hit d3 = %+v, want it still returned with its edge", got)+ }++ // The ordinary (no-keyword) read is unchanged: depth-0 clusters only.+ clusters, rootTotal, err := ns.ListClusters(t.Context(), "t1", "kb1", "", 0, 100)+ if err != nil {+ t.Fatal(err)+ }+ if rootTotal != 1 || len(clusters) != 1 || clusters[0].Name != rootCluster {+ t.Fatalf("root listing = %+v (total %d), want the single root cluster", clusters, rootTotal)+ }+ if clusters[0].Matched {+ t.Error("a non-search read must not mark nodes as hits")+ }+}++// TestSearchNavClusterCount pins the search header's number: how many clusters+// the hits landed in — not the hit count (which would make the header mean+// "documents" in search mode), and not every cluster row returned either, since+// an ancestor shared by two hits is a path row.+func TestSearchNavClusterCount(t *testing.T) {+ hit := func(docID, parent string) map[string]interface{} {+ return map[string]interface{}{"doc_id": docID, "parent_kwd": parent}+ }+ cases := []struct {+ name string+ matches []map[string]interface{}+ want int64+ }{+ {+ name: "no hits",+ matches: nil,+ want: 0,+ },+ {+ name: "two hits in one cluster count once",+ matches: []map[string]interface{}{hit("d1", "三国 11111111"), hit("d2", "三国 11111111")},+ want: 1,+ },+ {+ name: "two sub-clusters under one root count their own clusters, not the ancestor",+ matches: []map[string]interface{}{+ hit("d1", "名将 22222222"), hit("d2", "谋士 33333333"),+ },+ want: 2,+ },+ {+ name: "hits in different root clusters",+ matches: []map[string]interface{}{hit("d1", "三国 11111111"), hit("d2", "唐宋 44444444")},+ want: 2,+ },+ {+ name: "a leaf with no cluster edge is not counted",+ matches: []map[string]interface{}{hit("d1", navRootParent), hit("d2", "")},+ want: 0,+ },+ {+ name: "an empty row is dropped",+ matches: []map[string]interface{}{{}, hit("d1", "三国 11111111")},+ want: 1,+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := searchNavClusterCount(tc.matches); got != tc.want {+ t.Errorf("searchNavClusterCount = %d, want %d", got, tc.want)+ }+ })+ }}// TestNavNamingHelpers_Readable verifies the nav display-name helpers produce@@ -436,6 +645,48 @@ func TestNodeFromRow_ReadableName(t *testing.T) {}}+// TestNodeFromRow_LabelSource pins which column the nav tree renders: title_kwd+// (the summary's first line, the same label NavHit.Name carries to the model),+// overridden by an explicit readable `name` column when the row has one (Python's+// writer stores one; Go's rows do not). A cluster keeps its key from title_kwd.+func TestNodeFromRow_LabelSource(t *testing.T) {+ ns := &NavService{}+ cases := []struct {+ name string+ row map[string]interface{}+ want string+ }{+ {+ name: "the content title labels a document leaf",+ row: map[string]interface{}{+ "title_kwd": "三国名将事迹概要", "type_kwd": "nav_doc",+ },+ want: "三国名将事迹概要",+ },+ {+ name: "the name column wins over the content title",+ row: map[string]interface{}{+ "name": "三国名将事迹概要", "title_kwd": "机器键_ab12cd34ef56", "type_kwd": "nav_doc",+ },+ want: "三国名将事迹概要",+ },+ {+ name: "a cluster keeps its key from title_kwd",+ row: map[string]interface{}{+ "title_kwd": "三国名将事迹概要 3fdf14d0", "type_kwd": "nav_cluster", "doc_count_int": 3,+ },+ want: "三国名将事迹概要 3fdf14d0",+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := ns.nodeFromRow(tc.row, "doc").Name; got != tc.want {+ t.Errorf("nodeFromRow name = %q, want %q", got, tc.want)+ }+ })+ }+}+// TestNavService_Search_ReturnsHit asserts acceptance #5.func TestNavService_Search_ReturnsHit(t *testing.T) {eng := newMemNavEngine()@@ -706,6 +957,426 @@ func TestNavService_NavDocDepth(t *testing.T) {}}+// TestNavService_NavDocLabel pins the nav_doc leaf's label on both write sites:+// title_kwd carries the summary's first line, one column for both readers — the+// tree (nodeFromRow) and the model (NavHit.Name → the routed document's+// <summary>). No second label column is written.+func TestNavService_NavDocLabel(t *testing.T) {+ summary := "三国名将事迹概要\n三国名将事迹概述:关羽斩颜良诛文丑。"++ t.Run("the leaf label is the summary's first line", func(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", summary)); err != nil {+ t.Fatalf("UpsertDoc: %v", err)+ }+ if got := navDocRowTitle(t, eng, "d1"); got != "三国名将事迹概要" {+ t.Errorf("nav_doc title_kwd = %q, want the summary's first line", got)+ }+ if _, ok := navDocRow(t, eng, "d1")["docnm_kwd"]; ok {+ t.Error("nav_doc carries a separate docnm_kwd label; the label lives in title_kwd")+ }+ })++ t.Run("the merge path labels the leaf too", func(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ // d1 creates the root cluster; d2's summary embeds identically, so it+ // merges into that cluster (the second nav_doc write site).+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", "aaa one")); err != nil {+ t.Fatalf("UpsertDoc d1: %v", err)+ }+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d2", "aaa two")); err != nil {+ t.Fatalf("UpsertDoc d2: %v", err)+ }+ if got := navDocRowTitle(t, eng, "d2"); got != "aaa two" {+ t.Errorf("merged nav_doc title_kwd = %q, want the summary's first line", got)+ }+ })+}++// TestNavService_UpsertDoc_RestoresContentLabel covers the recompile upgrade for+// legacy rows whose title_kwd holds the document file name (written by the+// earlier label rule): the refresh puts the leaf back on the summary's first+// line, in place (same row).+func TestNavService_UpsertDoc_RestoresContentLabel(t *testing.T) {+ summary := "三国名将事迹概要\n三国名将事迹概述:关羽斩颜良诛文丑。"+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ in := navUpsertInput("t1", "kb1", "d1", summary)+ if err := ns.UpsertDoc(t.Context(), in); err != nil {+ t.Fatalf("UpsertDoc: %v", err)+ }+ // Simulate a legacy row whose leaf label holds the file name.+ navDocRow(t, eng, "d1")["title_kwd"] = "三国人物.pdf"++ if err := ns.UpsertDoc(t.Context(), in); err != nil {+ t.Fatalf("re-upsert: %v", err)+ }+ if got := navDocRowTitle(t, eng, "d1"); got != "三国名将事迹概要" {+ t.Errorf("nav_doc title_kwd = %q, want it restored to the summary's first line", got)+ }+ assertSingleNavDocRow(t, eng, "d1")+}++// TestNavService_UpsertDoc_PlacementBySimilarity pins where a similar-but-not-+// mergeable document opens its cluster, against the three similarity bands+// (Python _MERGE_THRESHOLD=0.80 / _MIN_SIM=0.50):+//+// - >= 0.80 → merge into the best cluster (the document leaf joins it);+// - < 0.80 → its own cluster, at the ROOT level when the best match is a+// root cluster (Python attaches it to the best cluster's parent, which is+// the root sentinel there), one level deeper when the best match is nested.+//+// The stub embedder's vectors are nearly collinear for any text, so the bands+// are driven with explicit vectors (UpsertDocInput.Embedd) and seeded cluster+// rows.+func TestNavService_UpsertDoc_PlacementBySimilarity(t *testing.T) {+ const dim = 1024++ navVec := navTestUnitVec+ seedCluster := func(eng *memNavEngine, name, parentKwd string, depth int, vec []float64) {+ row := map[string]interface{}{+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavCluster,+ "title_kwd": name, "parent_kwd": parentKwd, "depth_int": depth,+ "doc_count_int": 0, "content_with_weight": `{"type":"nav_cluster","description":"topic"}`,+ "q_1024_vec": vec,+ }+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1"); err != nil {+ t.Fatal(err)+ }+ }+ findCluster := func(t *testing.T, eng *memNavEngine, parentKwd string) map[string]interface{} {+ t.Helper()+ for _, row := range eng.rows {+ if row["type_kwd"] == nav.TypeNavCluster && row["parent_kwd"] == parentKwd {+ return row+ }+ }+ t.Fatalf("no nav_cluster row with parent_kwd=%q", parentKwd)+ return nil+ }+ docRows := func(eng *memNavEngine, docID string) []map[string]interface{} {+ out := []map[string]interface{}{}+ for _, row := range eng.rows {+ if row["type_kwd"] == nav.TypeNavDoc && row["doc_id"] == docID {+ out = append(out, row)+ }+ }+ return out+ }++ t.Run("an unrelated doc opens a root-level cluster", func(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ seedCluster(eng, "三国演义 aaaaaaaa", navRootParent, 0, navVec(1))++ in := navUpsertInput("t1", "kb1", "d2", "出师表要义")+ in.Embedd = f64ToF32Slice(navVec(0.60, 0.80)) // cos 0.60: related, not mergeable+ if err := ns.UpsertDoc(t.Context(), in); err != nil {+ t.Fatalf("UpsertDoc: %v", err)+ }++ // The seeded cluster keeps its name; the newly opened one is the other+ // root-level cluster.+ var opened map[string]interface{}+ for _, row := range eng.rows {+ if row["type_kwd"] == nav.TypeNavCluster && row["title_kwd"] != "三国演义 aaaaaaaa" {+ opened = row+ }+ }+ if opened == nil {+ t.Fatal("no root-level cluster was opened for the unrelated document")+ }+ if opened["parent_kwd"] != navRootParent {+ t.Errorf("new cluster parent_kwd = %v, want the root sentinel (top level, beside the existing cluster)", opened["parent_kwd"])+ }+ if d, _ := opened["depth_int"].(int); d != 0 {+ t.Errorf("new cluster depth_int = %v, want 0 (a root cluster)", opened["depth_int"])+ }+ // Its document leaf sits one level below it.+ rows := docRows(eng, "d2")+ if len(rows) != 1 {+ t.Fatalf("nav_doc rows for d2 = %d, want 1", len(rows))+ }+ if rows[0]["parent_kwd"] != opened["title_kwd"] {+ t.Errorf("nav_doc parent = %v, want the new cluster %v", rows[0]["parent_kwd"], opened["title_kwd"])+ }+ if d, _ := rows[0]["depth_int"].(int); d != 1 {+ t.Errorf("nav_doc depth_int = %v, want 1", rows[0]["depth_int"])+ }+ })++ t.Run("a mergeable doc joins the best cluster", func(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ seedCluster(eng, "三国演义 aaaaaaaa", navRootParent, 0, navVec(1))++ in := navUpsertInput("t1", "kb1", "d2", "三国名将")+ in.Embedd = f64ToF32Slice(navVec(0.90, math.Sqrt(1-0.81))) // cos 0.90+ if err := ns.UpsertDoc(t.Context(), in); err != nil {+ t.Fatalf("UpsertDoc: %v", err)+ }+ rows := docRows(eng, "d2")+ if len(rows) != 1 || rows[0]["parent_kwd"] != "三国演义 aaaaaaaa" {+ t.Fatalf("nav_doc rows = %+v, want the leaf under the merged cluster", rows)+ }+ if clusters := countNavClusters(eng); clusters != 1 {+ t.Errorf("nav_cluster rows = %d, want 1 (a merge must not open a cluster)", clusters)+ }+ })++ t.Run("a nested best match keeps the child cluster", func(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ // Root cluster at cos 0.70 with the doc (>= _RECURSE_THRESHOLD, so the+ // descent continues), nested cluster at 0.78 (< merge threshold, > the+ // root's, so IT becomes the best match).+ seedCluster(eng, "三国演义 aaaaaaaa", navRootParent, 0, navVec(1))+ nested := "出师表要义 bbbbbbbb"+ seedCluster(eng, nested, "三国演义 aaaaaaaa", 1, navVec(0.546, 0.5569, 0.6257))++ in := navUpsertInput("t1", "kb1", "d3", "出师表陈情")+ in.Embedd = f64ToF32Slice(navVec(0.70, 0.714)) // cos(root) 0.70, cos(child) 0.78+ if err := ns.UpsertDoc(t.Context(), in); err != nil {+ t.Fatalf("UpsertDoc: %v", err)+ }+ child := findCluster(t, eng, nested)+ if d, _ := child["depth_int"].(int); d != 2 {+ t.Errorf("new child cluster depth_int = %v, want 2 (best depth 1 + 1)", child["depth_int"])+ }+ rows := docRows(eng, "d3")+ if len(rows) != 1 || rows[0]["parent_kwd"] != child["title_kwd"] {+ t.Fatalf("nav_doc rows = %+v, want the leaf under the new child cluster %v", rows, child["title_kwd"])+ }+ if d, _ := rows[0]["depth_int"].(int); d != 3 {+ t.Errorf("nav_doc depth_int = %v, want 3", rows[0]["depth_int"])+ }+ })++}++// navTestUnitVec builds a 1024-dim unit vector from the leading components (the+// rest zero), so a test can dial an exact cosine against a seeded cluster's+// vector. 1024 is the stub embedder's dimension.+func navTestUnitVec(components ...float64) []float64 {+ v := make([]float64, 1024)+ var norm float64+ for i, c := range components {+ v[i] = c+ norm += c * c+ }+ if norm > 0 {+ scale := 1 / math.Sqrt(norm)+ for i := range v {+ v[i] *= scale+ }+ }+ return v+}++// TestNavService_FindBestCluster_DeepestMatchWins pins the descent result against+// Python _find_best_cluster: the returned match is the DEEPEST cluster reached+// while every step clears navRecurse (0.65) — not the strongest one seen — and a+// child below the threshold is not adopted at all. The placement branch decides+// from that (deeper, possibly lower) similarity, so this is what keeps the two+// implementations in step.+func TestNavService_FindBestCluster_DeepestMatchWins(t *testing.T) {+ const (+ rootName = "三国演义 aaaaaaaa"+ childName = "出师表要义 bbbbbbbb"+ )+ seedCluster := func(eng *memNavEngine, name, parentKwd string, depth int, vec []float64) {+ row := map[string]interface{}{+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavCluster,+ "title_kwd": name, "parent_kwd": parentKwd, "depth_int": depth,+ "doc_count_int": 0, "content_with_weight": `{"type":"nav_cluster","description":"topic"}`,+ "q_1024_vec": vec,+ }+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{row}, "", "kb1"); err != nil {+ t.Fatal(err)+ }+ }+ // doc/child pairs are built so the document's cosine against each seeded+ // cluster vector is exactly the intended value.+ cases := []struct {+ name string+ doc []float64+ child []float64+ wantName string+ wantSim float64+ wantDepth int+ }{+ {+ // Root 0.85 (mergeable on its own) but the child still clears the+ // recurse threshold at 0.70: Python descends and returns the CHILD,+ // so the placement sees 0.70 and opens a new cluster instead of+ // merging into the root.+ name: "a recurseable descendant displaces a stronger ancestor",+ doc: navTestUnitVec(0.85, 0.5268),+ child: navTestUnitVec(0.595, 0.3688, 0.7141),+ wantName: childName,+ wantSim: 0.70,+ wantDepth: 1,+ },+ {+ // The child is below the threshold: it is not adopted at all, so the+ // root (0.90) stays the answer and the document merges into it.+ name: "a child below the recurse threshold is not adopted",+ doc: navTestUnitVec(0.90, 0.4359),+ child: navTestUnitVec(0.36, 0.1744, 0.9165),+ wantName: rootName,+ wantSim: 0.90,+ wantDepth: 0,+ },+ {+ // The root itself is returned even when it does not clear the+ // threshold (Python seeds `best` with its top-1 root).+ name: "an unrelated root is still returned",+ doc: navTestUnitVec(0.40, 0.9165),+ child: navTestUnitVec(0.36, 0.1744, 0.9165),+ wantName: rootName,+ wantSim: 0.40,+ wantDepth: 0,+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ seedCluster(eng, rootName, navRootParent, 0, navTestUnitVec(1))+ seedCluster(eng, childName, rootName, 1, tc.child)++ name, sim, depth, err := ns.findBestCluster(t.Context(), "t1", "kb1", f64ToF32Slice(tc.doc))+ if err != nil {+ t.Fatalf("findBestCluster: %v", err)+ }+ if name != tc.wantName || depth != tc.wantDepth {+ t.Errorf("findBestCluster = (%q, %v, %d), want (%q, %v, %d)",+ name, sim, depth, tc.wantName, tc.wantSim, tc.wantDepth)+ }+ if math.Abs(sim-tc.wantSim) > 1e-4 {+ t.Errorf("findBestCluster sim = %v, want %v", sim, tc.wantSim)+ }+ })+ }+}++// countNavClusters counts the nav_cluster rows of the mem engine.+func countNavClusters(eng *memNavEngine) int {+ n := 0+ for _, row := range eng.rows {+ if row["type_kwd"] == nav.TypeNavCluster {+ n+++ }+ }+ return n+}++// f64ToF32Slice narrows an explicit test vector to the engine's float32 shape.+func f64ToF32Slice(in []float64) []float32 {+ out := make([]float32, len(in))+ for i, v := range in {+ out[i] = float32(v)+ }+ return out+}++// assertSingleNavDocRow fails when docID has anything other than one nav_doc+// row: the label refresh must UPDATE, never duplicate the leaf.+func assertSingleNavDocRow(t *testing.T, eng *memNavEngine, docID string) {+ t.Helper()+ count := 0+ for _, row := range eng.rows {+ if row["type_kwd"] == "nav_doc" && row["doc_id"] == docID {+ count+++ }+ }+ if count != 1 {+ t.Errorf("nav_doc rows for %s = %d, want 1 (refresh, not a duplicate)", docID, count)+ }+}++// TestNavService_SummariesByDocIDs_DescriptionLabel pins the routed-document+// label — which callers render to the model as the document's <summary> — to the+// nav_doc's stored description, i.e. the document's own summary text (Python+// _nav_doc_summaries). The row's label is only the fallback: title_kwd carries+// the summary's first line, not the whole text.+func TestNavService_SummariesByDocIDs_DescriptionLabel(t *testing.T) {+ summary := "三国名将事迹概要\n三国名将事迹概述:关羽斩颜良诛文丑。"++ t.Run("the description wins over the tree label", func(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", summary)); err != nil {+ t.Fatalf("UpsertDoc: %v", err)+ }+ got := ns.SummariesByDocIDs(t.Context(), "t1", "kb1", []string{"d1"})+ if got["d1"] != summary {+ t.Errorf("routed label = %q, want the stored description", got["d1"])+ }+ if got["d1"] == navDocRowTitle(t, eng, "d1") {+ t.Error("routed label fell back to the tree label; the description must win")+ }+ })++ t.Run("a row without a description falls back to its label", func(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{+ {+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,+ "doc_id": "d9", "title_kwd": "出师表全文内容与赏析",+ "content_with_weight": `{"type":"nav_doc"}`,+ },+ }, "", "kb1"); err != nil {+ t.Fatal(err)+ }+ got := ns.SummariesByDocIDs(t.Context(), "t1", "kb1", []string{"d9"})+ if got["d9"] != "出师表全文内容与赏析" {+ t.Errorf("label = %q, want the stored title_kwd fallback", got["d9"])+ }+ })++ t.Run("a raw id is never a label", func(t *testing.T) {+ const rawID = "6564be8a6564be8a6564be8a6564be8a"+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ if _, err := eng.InsertChunks(t.Context(), []map[string]interface{}{+ {+ "compile_kwd": navCompileKwd, "type_kwd": nav.TypeNavDoc,+ "doc_id": rawID, "title_kwd": rawID,+ "content_with_weight": `{"type":"nav_doc"}`,+ },+ }, "", "kb1"); err != nil {+ t.Fatal(err)+ }+ got := ns.SummariesByDocIDs(t.Context(), "t1", "kb1", []string{rawID})+ if _, ok := got[rawID]; ok {+ t.Errorf("raw id used as a label: %+v", got)+ }+ })+}++// navDocRow returns the nav_doc row for docID.+func navDocRow(t *testing.T, eng *memNavEngine, docID string) map[string]interface{} {+ t.Helper()+ for _, row := range eng.rows {+ if row["type_kwd"] == "nav_doc" && row["doc_id"] == docID {+ return row+ }+ }+ t.Fatalf("no nav_doc row for doc %s", docID)+ return nil+}++// navDocRowTitle returns the title_kwd of the nav_doc row for docID.+func navDocRowTitle(t *testing.T, eng *memNavEngine, docID string) string {+ t.Helper()+ title, _ := navDocRow(t, eng, docID)["title_kwd"].(string)+ return title+}+// TestNavService_RemoveDoc_CascadesToEmptyCluster covers A3: after removing a// doc, a cluster left with no docs and no children is pruned (cleanupEmptyCluster),// and the doc is dropped from its parent cluster's doc_ids_kwd.@@ -847,6 +1518,159 @@ func TestNavService_MaybeSplitCluster_SplitsOverfull(t *testing.T) {}}+// stubNavMergeLLM is a canned nav.NavMergeLLM: it names every new cluster with+// the configured topic title and fuses merges into the configured text.+type stubNavMergeLLM struct {+ name string+ summary string+ merged string+ mergeSeen [][]string+}++func (s *stubNavMergeLLM) CreateSummary(_ context.Context, _, _ string) (string, string, error) {+ return s.name, s.summary, nil+}++func (s *stubNavMergeLLM) Merge(_ context.Context, _ string, texts []string) (string, error) {+ s.mergeSeen = append(s.mergeSeen, texts)+ return s.merged, nil+}++// recordingNavEmbedder records every embedded text so a test can assert that a+// rewritten cluster description was re-embedded.+type recordingNavEmbedder struct {+ texts []string+}++func (e *recordingNavEmbedder) Encode(ctx context.Context, tenantID string, texts []string) ([][]float32, error) {+ e.texts = append(e.texts, texts...)+ return stubNavEmbedder{}.Encode(ctx, tenantID, texts)+}++// navClusterRow returns the first nav_cluster row of the mem engine.+func navClusterRow(t *testing.T, eng *memNavEngine) map[string]interface{} {+ t.Helper()+ for _, row := range eng.rows {+ if row["type_kwd"] == "nav_cluster" {+ return row+ }+ }+ t.Fatal("no nav_cluster row")+ return nil+}++// navRowDescription parses a nav row payload's description.+func navRowDescription(t *testing.T, row map[string]interface{}) string {+ t.Helper()+ raw, _ := row["content_with_weight"].(string)+ var m map[string]interface{}+ if err := json.Unmarshal([]byte(raw), &m); err != nil {+ t.Fatalf("parse nav payload %q: %v", raw, err)+ }+ desc, _ := m["description"].(string)+ return desc+}++// TestNavService_LLMClusterNaming pins the wired naming path: the cluster takes+// the model's topic title as the title part and keeps the " <8-hex>" suffix —+// the name is also the cluster key (parent_kwd / row id), so it must stay unique+// per KB — and the model's summary becomes the cluster's description.+func TestNavService_LLMClusterNaming(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ ns.SetNavMergeLLM(&stubNavMergeLLM{name: "三国名将事迹", summary: "三国名将事迹概览。"})+ summary := "三国名将事迹概要\n正文"+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", summary)); err != nil {+ t.Fatalf("UpsertDoc: %v", err)+ }+ cluster := navClusterRow(t, eng)+ if want := readableClusterName("三国名将事迹", summary); cluster["title_kwd"] != want {+ t.Errorf("cluster title_kwd = %v, want %q (topic title + uniqueness suffix)", cluster["title_kwd"], want)+ }+ if got := navRowDescription(t, cluster); got != "三国名将事迹概览。" {+ t.Errorf("cluster description = %q, want the model summary", got)+ }+}++// TestNavService_MergeFusesClusterDescription pins the wired merge path: a+// document that joins an existing cluster fuses its summary into the cluster+// description (Python _llm_merge) and re-embeds the rewritten description, so+// later placements score against the merged topic.+func TestNavService_MergeFusesClusterDescription(t *testing.T) {+ eng := newMemNavEngine()+ embedder := &recordingNavEmbedder{}+ ns := NewNavService(embedder)+ ns.engine = eng+ stub := &stubNavMergeLLM{name: "topic", summary: "first doc summary", merged: "fused description"}+ ns.SetNavMergeLLM(stub)++ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", "aaa one")); err != nil {+ t.Fatalf("UpsertDoc d1: %v", err)+ }+ // d2's summary embeds identically under the deterministic stub embedder, so+ // it is placed inside d1's cluster (the merge branch).+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d2", "aaa two")); err != nil {+ t.Fatalf("UpsertDoc d2: %v", err)+ }++ clusters := 0+ for _, row := range eng.rows {+ if row["type_kwd"] == "nav_cluster" {+ clusters+++ }+ }+ if clusters != 1 {+ t.Fatalf("nav_cluster rows = %d, want 1 (d2 must merge, not create a cluster)", clusters)+ }+ cluster := navClusterRow(t, eng)+ if got := navRowDescription(t, cluster); got != "fused description" {+ t.Errorf("cluster description = %q, want the merged text", got)+ }+ if ids := firstStringSlice(cluster["doc_ids_kwd"]); len(ids) != 2 {+ t.Errorf("cluster doc_ids_kwd = %v, want both documents", ids)+ }+ if len(stub.mergeSeen) != 1 {+ t.Fatalf("Merge calls = %d, want 1", len(stub.mergeSeen))+ }+ if texts := stub.mergeSeen[0]; len(texts) != 2 || texts[0] != "first doc summary" || texts[1] != "aaa two" {+ t.Errorf("Merge texts = %v, want [existing description, new summary]", texts)+ }+ reEmbedded := false+ for _, text := range embedder.texts {+ if text == "fused description" {+ reEmbedded = true+ break+ }+ }+ if !reEmbedded {+ t.Errorf("merged description was not re-embedded; embedded texts = %v", embedder.texts)+ }+}++// TestNavService_MergeKeepsDescriptionWithoutLLM pins the no-LLM fallback:+// without a NavMergeLLM the cluster keeps the description it was created with+// (Python _llm_merge returns the existing text when chat_mdl is nil) instead of+// accumulating a concatenation of every document summary.+func TestNavService_MergeKeepsDescriptionWithoutLLM(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", "aaa one")); err != nil {+ t.Fatalf("UpsertDoc d1: %v", err)+ }+ cluster := navClusterRow(t, eng)+ before := navRowDescription(t, cluster)++ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d2", "aaa two")); err != nil {+ t.Fatalf("UpsertDoc d2: %v", err)+ }+ if got := navRowDescription(t, cluster); got != before {+ t.Errorf("cluster description = %q, want it unchanged (%q) without an LLM", got, before)+ }+ if ids := firstStringSlice(cluster["doc_ids_kwd"]); len(ids) != 2 {+ t.Errorf("cluster doc_ids_kwd = %v, want both documents", ids)+ }+}+// sTestNavIndex returns the nav index name for a test NavService (tenant "t1").func sTestNavIndex(ns *NavService) string {return "ragflow_t1"diff --git a/web/src/interfaces/database/dataset-nav.ts b/web/src/interfaces/database/dataset-nav.tsindex 7916a2b995..69f45743da 100644--- a/web/src/interfaces/database/dataset-nav.ts+++ b/web/src/interfaces/database/dataset-nav.ts@@ -8,9 +8,22 @@ export interface DatasetNavNode {keywords?: string[];entities?: string[];graph_content?: string;+ /**+ * Tree edge: the parent cluster's name ("root" for a depth-0 cluster). A+ * keyword search returns the hits together with the cluster path above them,+ * so the flat list nests into one tree per root cluster.+ */+ parent_kwd?: string;+ /** Keyword-search hit marker; the path rows returned with it are not hits. */+ matched?: boolean;}export interface DatasetNavList {+ /**+ * Cluster count: the depth-0 clusters when browsing, or the clusters the hits+ * landed in when searching. The nav tree header shows it in both modes, so a+ * search never re-purposes that heading as a document count.+ */total: number;items: DatasetNavNode[];}diff --git a/web/src/pages/dataset/compilation/hooks/use-compilation-nav.ts b/web/src/pages/dataset/compilation/hooks/use-compilation-nav.tsindex 82c35e8e0f..90011d3274 100644--- a/web/src/pages/dataset/compilation/hooks/use-compilation-nav.ts+++ b/web/src/pages/dataset/compilation/hooks/use-compilation-nav.ts@@ -66,12 +66,11 @@ export function useCompilationNav() {const { data: childrenData, isError: childrenError } =useFetchDatasetNavChildren(loadingParent, activeKeywords);+ // Opened documents get their FULL structure graph, without the nav keywords: a+ // keyword-filtered read would hide every entity that does not match (keyword+ // drill-down lives in the structure view's own search box).const { data: structureData, isPlaceholderData: structurePlaceholder } =- useFetchDocumentStructureGraphById(- kbId,- loadingDocId ?? '',- activeKeywords || undefined,- );+ useFetchDocumentStructureGraphById(kbId, loadingDocId ?? '');useEffect(() => {if (!loadingParent || !childrenData) {diff --git a/web/src/pages/dataset/compilation/nav-tree-left-panel.tsx b/web/src/pages/dataset/compilation/nav-tree-left-panel.tsxindex f725ba476d..d27779c156 100644--- a/web/src/pages/dataset/compilation/nav-tree-left-panel.tsx+++ b/web/src/pages/dataset/compilation/nav-tree-left-panel.tsx@@ -18,6 +18,12 @@ import { useTranslation } from 'react-i18next';import { UpdateLogSheet } from './update-log-sheet';import { buildNavTreeData, NavEntityClickHandler } from './utils/nav-tree';+// TreeView only computes expandedItemIds when initialSelectedItemId is truthy;+// combined with expandAll, any truthy id makes every branch mount open. A+// sentinel that matches no real node forces expand-all without highlighting any+// row as selected (same trick as skills-left-panel).+const NavExpandAllSentinelId = '__nav-tree-expand-all-sentinel__';+type NavNodeDeleteActionProps = {name: string;parentName: string | null;@@ -152,6 +158,10 @@ export function NavTreeLeftPanel({childrenMap,childrenErrorParents,structureMap,+ // A search response is a pruned forest (hits + the cluster path above+ // them), so it is nested from the payload instead of being fetched+ // branch by branch.+ searchMode: !!activeKeywords,getActions: renderNavActions,onNodeClick,onNodeExpand,@@ -161,6 +171,7 @@ export function NavTreeLeftPanel({}),[navList?.items,+ activeKeywords,childrenMap,childrenErrorParents,structureMap,@@ -244,6 +255,11 @@ export function NavTreeLeftPanel({<TreeViewkey={activeKeywords}data={treeData}+ // Search: mount the matched branches open (sentinel trick above).+ expandAll={!!activeKeywords}+ initialSelectedItemId={+ activeKeywords ? NavExpandAllSentinelId : undefined+ }expandOnRowClick={false}defaultNodeIcon={Folder}defaultLeafIcon={FileText}diff --git a/web/src/pages/dataset/compilation/utils/nav-tree.test.ts b/web/src/pages/dataset/compilation/utils/nav-tree.test.tsindex 7ff388f7aa..e7d2c8af68 100644--- a/web/src/pages/dataset/compilation/utils/nav-tree.test.ts+++ b/web/src/pages/dataset/compilation/utils/nav-tree.test.ts@@ -1,4 +1,7 @@-import { buildNavTreeData } from './nav-tree';+import { adaptTreeToTreeData } from '@/components/structure-graph/adapters';+import { CompilationTemplateKind } from '@/constants/compilation';++import { buildNavTreeData, nestNavSearchHits } from './nav-tree';jest.mock('@/components/structure-graph/adapters', () => ({adaptPageIndexToTreeData: jest.fn(() => []),@@ -46,3 +49,169 @@ describe('buildNavTreeData', () => {expect(data[0].children).toBeUndefined();});});++function doc(docId: string, name: string, parentKwd: string, matched = true) {+ return {+ name,+ description: '',+ doc_count: 1,+ type: 'doc',+ doc_id: docId,+ has_children: false,+ parent_kwd: parentKwd,+ matched,+ };+}++function clusterIn(name: string, parentKwd: string, docCount = 1) {+ return {+ name,+ description: '',+ doc_count: docCount,+ type: 'cluster',+ has_children: true,+ parent_kwd: parentKwd,+ matched: false,+ };+}++describe('nestNavSearchHits', () => {+ it('turns the flat hits plus their cluster path into a forest', () => {+ const { roots, children } = nestNavSearchHits([+ clusterIn('三国 11111111', 'root'),+ clusterIn('名将 22222222', '三国 11111111'),+ doc('d1', '三国人物.pdf', '名将 22222222'),+ doc('d2', '其他.pdf', '三国 11111111'),+ ]);++ expect(roots.map((node) => node.name)).toEqual(['三国 11111111']);+ expect(children['cluster:三国 11111111'].map((node) => node.name)).toEqual([+ '名将 22222222',+ '其他.pdf',+ ]);+ expect(children['cluster:名将 22222222'].map((node) => node.name)).toEqual([+ '三国人物.pdf',+ ]);+ });++ it('keeps a hit whose parent is not in the result set as its own root', () => {+ const { roots, children } = nestNavSearchHits([+ doc('d9', '孤儿.pdf', 'missing-cluster'),+ ]);++ expect(roots.map((node) => node.name)).toEqual(['孤儿.pdf']);+ expect(children).toEqual({});+ });+});++describe('buildNavTreeData (search mode)', () => {+ const SearchOptions = { ...TreeOptions, childrenMap: {}, searchMode: true };++ it('renders ONE tree per root cluster holding only the matched branches', () => {+ const data = buildNavTreeData(+ [+ clusterIn('三国 11111111', 'root', 2),+ clusterIn('名将 22222222', '三国 11111111'),+ doc('d1', '三国人物.pdf', '名将 22222222'),+ doc('d2', '其他.pdf', '三国 11111111'),+ ],+ SearchOptions,+ );++ expect(data).toHaveLength(1);+ expect(data[0].id).toBe('cluster:三国 11111111');+ expect(data[0].hasChildren).toBe(true);+ // Child ids are prefixed with the parent's id so they stay unique across+ // the forest.+ expect(data[0].children?.map((child) => child.id)).toEqual([+ 'cluster:三国 11111111/cluster:名将 22222222',+ 'cluster:三国 11111111/doc:d2',+ ]);+ // The deeper match sits under its matched ancestor, not beside it.+ expect(data[0].children?.[0].children?.map((child) => child.id)).toEqual([+ 'cluster:三国 11111111/cluster:名将 22222222/doc:d1',+ ]);+ });++ it('does not mount a pending structure placeholder under a matched leaf', () => {+ // The branches mount already expanded, so a placeholder would never be+ // replaced (mount-time expansion fires no onExpand, which fetches the graph).+ const data = buildNavTreeData([doc('d1', '三国人物.pdf', 'root')], {+ ...SearchOptions,+ structureMap: {},+ });++ expect(data).toHaveLength(1);+ expect(data[0].children).toBeUndefined();+ expect(data[0].onExpand).toEqual(expect.any(Function));+ });++ it('mounts the fetched structure graph under a matched leaf', () => {+ jest+ .mocked(adaptTreeToTreeData)+ .mockReturnValueOnce([+ { id: 'e1', name: '刘备', children: [{ id: 'e2', name: '关羽' }] },+ ]);+ const onEntityClick = jest.fn();+ const data = buildNavTreeData([doc('d1', '三国人物.pdf', 'root')], {+ ...SearchOptions,+ onEntityClick,+ structureMap: {+ d1: [+ {+ template_id: 't1',+ template_name: '结构树',+ kind: CompilationTemplateKind.Tree,+ entities: [+ { id: 'e1', name: '刘备', description: '蜀汉开国皇帝' },+ { id: 'e2', name: '关羽' },+ ],+ relations: [],+ },+ ],+ },+ });++ expect(data).toHaveLength(1);+ expect(data[0].hasChildren).toBe(true);+ // Adapter ids are re-prefixed as <document id>/<template id>/<entity id>, so+ // they stay unique across the forest, and the hit stays expandable.+ expect(data[0].children?.map((child) => child.id)).toEqual([+ 'doc:d1/t1/e1',+ ]);+ expect(data[0].children?.[0].children?.map((child) => child.id)).toEqual([+ 'doc:d1/t1/e2',+ ]);+ expect(data[0].onExpand).toEqual(expect.any(Function));+ // Clicking an entity reports its document, display name and description —+ // the same payload the non-search tree sends.+ data[0].children?.[0].onClick?.();+ expect(onEntityClick).toHaveBeenCalledWith(+ expect.objectContaining({ doc_id: 'd1' }),+ '刘备',+ '蜀汉开国皇帝',+ );+ });++ it('leaves the matched leaf a leaf when the fetched graph holds no entity', () => {+ // The adapter is mocked to return no items, so a fetched-but-empty graph+ // renders no children instead of an empty expandable branch.+ const data = buildNavTreeData([doc('d1', '三国人物.pdf', 'root')], {+ ...SearchOptions,+ structureMap: {+ d1: [+ {+ template_id: 't1',+ template_name: '结构树',+ kind: CompilationTemplateKind.Tree,+ entities: [],+ relations: [],+ },+ ],+ },+ });++ expect(data[0].children).toBeUndefined();+ expect(data[0].onExpand).toEqual(expect.any(Function));+ });+});diff --git a/web/src/pages/dataset/compilation/utils/nav-tree.ts b/web/src/pages/dataset/compilation/utils/nav-tree.tsindex 923d79ca39..80e11317e9 100644--- a/web/src/pages/dataset/compilation/utils/nav-tree.ts+++ b/web/src/pages/dataset/compilation/utils/nav-tree.ts@@ -28,6 +28,12 @@ type BuildNavTreeDataOptions = {childrenMap: Record<string, DatasetNavNode[]>;childrenErrorParents?: Record<string, boolean>;structureMap: Record<string, IStructureGraphTemplate[]>;+ /**+ * Search mode: the items are a pruned forest — the matched document leaves+ * plus the cluster path above each of them, every row carrying parent_kwd —+ * so branches come from the payload instead of a lazy /children request.+ */+ searchMode?: boolean;getActions?: NavTreeActionsFactory;onNodeClick: (node: DatasetNavNode, parentName: string | null) => void;onNodeExpand: (node: DatasetNavNode) => void;@@ -117,20 +123,153 @@ function buildStructureTreeData(});}+// navNodeIdentity is the forest's node key. Identity, not display name: two+// documents may share a name, and a cluster's display name is exactly the key+// its children reference through parent_kwd.+function navNodeIdentity(node: DatasetNavNode): string {+ return node.type === 'doc' && node.doc_id+ ? `doc:${node.doc_id}`+ : `cluster:${node.name}`;+}++type NavSearchForest = {+ roots: DatasetNavNode[];+ children: Record<string, DatasetNavNode[]>;+};++// nestNavSearchHits groups a search response into the tree its rows describe: a+// row whose parent edge points at another returned row becomes that row's child,+// everything else is a root. The backend returns every hit together with the+// cluster path above it, so this yields ONE tree per root cluster holding only+// the matched branches — instead of one top-level tree per hit, where a matching+// cluster's own matching subtrees showed up as trees beside it.+export function nestNavSearchHits(+ items: DatasetNavNode[] = [],+): NavSearchForest {+ const present = new Set(items.map(navNodeIdentity));+ const children: Record<string, DatasetNavNode[]> = {};+ const roots: DatasetNavNode[] = [];+ items.forEach((node) => {+ const parentKey = node.parent_kwd ? `cluster:${node.parent_kwd}` : '';+ if (parentKey && present.has(parentKey)) {+ const bucket = children[parentKey];+ if (bucket) {+ bucket.push(node);+ } else {+ children[parentKey] = [node];+ }+ } else {+ roots.push(node);+ }+ });+ return { roots, children };+}++// mountDocumentStructure attaches a document leaf's structure-graph children to+// its tree item, or a loading placeholder while the graph is being fetched.+//+// The search path passes pendingPlaceholder false: its branches mount already+// expanded (expandAll), so a pending row under every hit would sit there for+// good — the mount-time expansion never fires onExpand, which is what triggers+// the fetch.+function mountDocumentStructure(+ item: TreeDataItem,+ node: DatasetNavNode,+ id: string,+ options: BuildNavTreeDataOptions,+ { pendingPlaceholder = true } = {},+): void {+ const templates = options.structureMap[node.doc_id as string];+ if (!templates) {+ // Not fetched yet: the placeholder keeps the node rendered as an expandable+ // branch until the request resolves.+ item.hasChildren = true;+ if (pendingPlaceholder) {+ item.children = [+ { id: `${id}/__loading__`, name: options.loadingPlaceholder },+ ];+ }+ return;+ }+ const children = buildStructureTreeData(templates, id, (entity) =>+ options.onEntityClick?.(+ node,+ getEntityDisplayName(entity),+ getEntityDescription(entity),+ ),+ );+ if (children.length) {+ item.hasChildren = true;+ item.children = children;+ }+ // No entity nodes: leave hasChildren unset so the node stays a leaf.+}++// buildNavSearchTreeData renders the pruned search forest. Branches come from+// the response, so expanding one fetches nothing; a matched document leaf still+// expands into its structure graph.+function buildNavSearchTreeData(+ nodes: DatasetNavNode[],+ forest: NavSearchForest,+ options: BuildNavTreeDataOptions,+ parentName: string | null,+ idPrefix: string,+): TreeDataItem[] {+ const { getActions, onNodeClick, onNodeExpand } = options;+ return nodes.map((node) => {+ const identity = navNodeIdentity(node);+ const id = idPrefix ? `${idPrefix}/${identity}` : identity;+ const item: TreeDataItem = {+ id,+ name: trim(node.name),+ actions: getActions?.(node, parentName),+ onClick: () => onNodeClick(node, parentName),+ };++ const branches = forest.children[identity];+ if (branches?.length) {+ item.hasChildren = true;+ item.children = buildNavSearchTreeData(+ branches,+ forest,+ options,+ node.name,+ id,+ );+ } else if (node.doc_id) {+ item.onExpand = () => onNodeExpand(node);+ mountDocumentStructure(item, node, id, options, {+ pendingPlaceholder: false,+ });+ }++ return item;+ });+}+export function buildNavTreeData(items: DatasetNavNode[] = [],options: BuildNavTreeDataOptions,parentName: string | null = null,idPrefix = '',): TreeDataItem[] {+ if (options.searchMode) {+ const forest = nestNavSearchHits(items);+ return buildNavSearchTreeData(+ forest.roots,+ forest,+ options,+ parentName,+ idPrefix,+ );+ }+const {childrenMap,childrenErrorParents,- structureMap,getActions,onNodeClick,onNodeExpand,- onEntityClick,loadingPlaceholder,errorPlaceholder,} = options;@@ -160,26 +299,7 @@ export function buildNavTreeData(// Fetched but empty: leave children unset so the branch opens to nothing.} else if (node.doc_id) {// Document leaf: expandable into its structure graph entities.- const templates = structureMap[node.doc_id];- if (!templates) {- // Not fetched yet: a placeholder keeps the node rendered as an- // expandable branch until the request resolves.- item.hasChildren = true;- item.children = [{ id: `${id}/__loading__`, name: loadingPlaceholder }];- } else {- const children = buildStructureTreeData(templates, id, (entity) =>- onEntityClick?.(- node,- getEntityDisplayName(entity),- getEntityDescription(entity),- ),- );- if (children.length) {- item.hasChildren = true;- item.children = children;- }- // No entity nodes: leave hasChildren unset so the node stays a leaf.- }+ mountDocumentStructure(item, node, id, options);}return item;