RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Port Python agentic search to Go (nav service, harness, tools) (#17702)
Zhichang Yu4e78f1f44092
diff --git a/.gitignore b/.gitignoreindex 1d3ab6baa0..521b9b9fa6 100644--- a/.gitignore+++ b/.gitignore@@ -222,12 +222,15 @@ backup/target-# Do not include in PR (local dev / build artifacts)+# Do not include in PR (local dev / build artifacts / working notes)ragflow.egg-info/uv-aarch64*.tar.gzuv-aarch64-unknown-linux-gnu.tar.gzdocker/launch_backend_service_windows.sh+# Scratch/working notes (agent task planning & self-review docs; not PR material)+tasks/*.md+# C++ build directoriesinternal/binding/cpp/build/internal/binding/cpp/cmake-build-release/@@ -239,6 +242,7 @@ internal/binding/cpp/cmake-build-debug/# Go server build outputbin/*!bin/.gitkeep+/ragflow_server.claude/settings.local.json.run/diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex 65910f5d0d..086bddd503 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -38,6 +38,7 @@ import (dataset "ragflow/internal/service/dataset""ragflow/internal/service/document""ragflow/internal/service/file"+ "ragflow/internal/service/nav""ragflow/internal/service/nlp""ragflow/internal/storage""ragflow/internal/syncer"@@ -835,6 +836,16 @@ func startServer(ctx context.Context) {compilationTemplateGroupHandler := handler.NewCompilationTemplateGroupHandler(service.NewCompilationTemplateGroupService())datasetArtifactHandler := handler.NewDatasetArtifactHandler(service.NewDatasetArtifactService(), datasetsService, file.NewFileCommitService())+ // Install the production eino-based chat invoker as the shared chat default,+ // so agentic-search harness LLM calls work in production. Without this,+ // chat.GetDefaultInvoker() stays nil and the harness falls back gracefully.+ component.InstallDefaultChatInvoker()++ // Install the dataset-nav ES-backed service (internal/service/nav ++ // internal/service/nlp). The embedder resolves the tenant's embedding model+ // on demand so Search/UpsertDoc can embed queries/summaries automatically.+ nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(modelProviderService, "")))+// Initialize routerr := router.NewRouter(authHandler,userHandler,diff --git a/internal/agent/chat/chat.go b/internal/agent/chat/chat.gonew file mode 100644index 0000000000..b8e3da96b4--- /dev/null+++ b/internal/agent/chat/chat.go@@ -0,0 +1,110 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// Package chat is a dependency-light home for the LLM chat-invoker interface+// and its package-level singleton. It lives here (leaf, importing only eino+// schema + gorm) so that both internal/agent/component (which owns the+// production eino-based invoker) and internal/agent/tool / internal/agent/harness+// (which need to call the LLM for routing/selection) can depend on it without+// forming an import cycle. The production invoker is registered at boot via+// SetDefaultInvoker.+package chat++import (+ "context"+ "sync"++ "github.com/cloudwego/eino/schema"+ "gorm.io/gorm"+)++// Invoker abstracts a chat-model call so callers can inject a stub in tests and+// production flows through the eino bridge. This is the shared seam for the LLM+// component and the agentic-search harness/tools.+type Invoker interface {+ Invoke(ctx context.Context, db *gorm.DB, req Request) (*Response, error)+}++// Request is the minimal surface needed to dispatch a chat call.+type Request struct {+ Driver string+ ModelName string+ APIKey string+ BaseURL string+ Messages []schema.Message+ Temperature *float64+ TopP *float64+ PresencePenalty *float64+ FrequencyPenalty *float64+ MaxTokens *int+ Thinking string // "enabled" | "disabled" | ""+}++// Response is the result of a chat call.+type Response struct {+ Content string+ Thinking string+ Model string+ Stopped bool+ Tokens int+}++// ErrNotConfigured is returned by GetDefaultInvoker when no production invoker+// has been installed. Callers should treat it as "no chat model available".+var ErrNotConfigured = &configError{"chat: default invoker not configured"}++type configError struct{ msg string }++func (e *configError) Error() string { return e.msg }++var (+ mu sync.RWMutex+ inst Invoker+ defaultModel string+)++// SetDefaultInvoker installs the (production or test) invoker. Pass nil to+// restore the "not configured" state.+func SetDefaultInvoker(inv Invoker) {+ mu.Lock()+ defer mu.Unlock()+ inst = inv+}++// SetDefaultModelName records the tenant-default chat model name. The production+// invoker uses it when a Request omits ModelName, so harness/agentic-search LLM+// calls work without threading model config through every node.+func SetDefaultModelName(name string) {+ mu.Lock()+ defer mu.Unlock()+ defaultModel = name+}++// GetDefaultModelName returns the configured default model name ("" if unset).+func GetDefaultModelName() string {+ mu.RLock()+ defer mu.RUnlock()+ return defaultModel+}++// GetDefaultInvoker returns the installed invoker. It returns nil when no+// invoker has been installed (e.g. before server bootstrap or in tests that do+// not need the LLM).+func GetDefaultInvoker() Invoker {+ mu.RLock()+ defer mu.RUnlock()+ return inst+}diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.goindex aaca4985d1..99720ce9c8 100644--- a/internal/agent/component/llm.go+++ b/internal/agent/component/llm.go@@ -18,12 +18,12 @@ import ("slices""sort""strings"- "sync""time""github.com/cloudwego/eino/schema""gorm.io/gorm"+ "ragflow/internal/agent/chat""ragflow/internal/agent/component/prompts""ragflow/internal/agent/runtime""ragflow/internal/common"@@ -130,84 +130,68 @@ type LLMOutput struct {Tokens int}-// ChatInvoker is the abstraction the LLM component uses to talk to a-// chat model. The default implementation lives in this file; tests can-// override the package-level defaultChatInvoker to inject a stub.-type ChatInvoker interface {- Invoke(ctx context.Context, db *gorm.DB, req ChatInvokeRequest) (*ChatInvokeResponse, error)-}+// ChatInvoker is an alias for the shared chat.Invoker seam. The production+// eino-based implementation lives in this file; the package-level singleton is+// owned by internal/agent/chat so agent tools and the harness can also call the+// LLM without an import cycle.+type ChatInvoker = chat.Invoker-// ChatInvokeRequest is the minimal surface the LLM component needs to-// dispatch a chat call. Driver / APIKey / ModelName are kept here so the-// invoker can wire the right provider without the component caring.-type ChatInvokeRequest struct {- Driver string- ModelName string- APIKey string- BaseURL string- Messages []schema.Message- Temperature *float64- TopP *float64- PresencePenalty *float64- FrequencyPenalty *float64- MaxTokens *int- // Thinking mirrors the agent-level `thinking` setting- // ("enabled" | "disabled" | ""). The default invoker is- // responsible for translating this into the provider-specific- // request body (e.g. Qwen `enable_thinking`, Kimi/GLM- // `thinking.type`). Empty string means "use provider default"- // and the invoker should leave the provider's reasoning mode- // untouched.- Thinking string-}+// ChatInvokeRequest is an alias for chat.Request.+type ChatInvokeRequest = chat.Request-// ChatInvokeResponse mirrors what the LLM component writes to its outputs.-type ChatInvokeResponse struct {- Content string- Thinking string- Model string- Stopped bool- Tokens int-}--// defaultChatInvokerMu guards defaultChatInvoker swaps during tests.-var defaultChatInvokerMu sync.RWMutex+// ChatInvokeResponse is an alias for chat.Response.+type ChatInvokeResponse = chat.Response-// defaultChatInvoker is the production ChatInvoker. Replaced in tests.-var defaultChatInvoker ChatInvoker = &einoChatInvoker{}--// SetDefaultChatInvoker swaps the package-level ChatInvoker (test helper).-// Pass nil to restore the default. Concurrent-safe.+// SetDefaultChatInvoker delegates to the shared chat package singleton (test+// helper). Pass nil to restore the "not configured" state. The production+// einoChatInvoker is registered at boot in cmd/server_main.go.func SetDefaultChatInvoker(inv ChatInvoker) {- defaultChatInvokerMu.Lock()- defer defaultChatInvokerMu.Unlock()- defaultChatInvoker = inv+ if inv == nil {+ chat.SetDefaultInvoker(nil)+ return+ }+ chat.SetDefaultInvoker(inv)+}++// GetDefaultChatInvokerForTest exposes the current shared chat invoker so+// cross-package tests can swap it and restore it safely.+func GetDefaultChatInvokerForTest() ChatInvoker {+ return chat.GetDefaultInvoker()}-// getDefaultChatInvoker returns the current default ChatInvoker.+// getDefaultChatInvoker returns the shared chat invoker, falling back to the+// production eino invoker when none has been installed.func getDefaultChatInvoker() ChatInvoker {- defaultChatInvokerMu.RLock()- defer defaultChatInvokerMu.RUnlock()- if defaultChatInvoker == nil {- return &einoChatInvoker{}+ if inv := chat.GetDefaultInvoker(); inv != nil {+ return inv}- return defaultChatInvoker+ return &einoChatInvoker{}}-// GetDefaultChatInvokerForTest exposes the current package-level invoker so-// cross-package tests can swap it and restore it safely.-func GetDefaultChatInvokerForTest() ChatInvoker {- return getDefaultChatInvoker()+// InstallDefaultChatInvoker registers the production eino-based invoker as the+// shared chat default. Called at server bootstrap so harness/agentic-search LLM+// calls work in production; without it, chat.GetDefaultInvoker() stays nil and+// harness falls back gracefully.+func InstallDefaultChatInvoker() {+ chat.SetDefaultInvoker(&einoChatInvoker{})}// einoChatInvoker is the production ChatInvoker — it constructs a fresh-// models.EinoChatModel per call from the request and dispatches.+// models.EinoChatModel per call from the request and dispatches. It is NOT+// registered as the shared chat default at init (so chat.GetDefaultInvoker()+// stays nil until bootstrap); cmd registers it via SetDefaultChatInvoker.type einoChatInvoker struct{}// Invoke satisfies ChatInvoker.func (e *einoChatInvoker) Invoke(ctx context.Context, db *gorm.DB, req ChatInvokeRequest) (*ChatInvokeResponse, error) {if req.ModelName == "" {- return nil, fmt.Errorf("component: LLM: model_id is required")+ // Harness/agentic-search nodes may omit the model; fall back to the+ // bootstrap-registered tenant default so those calls work in production.+ if def := chat.GetDefaultModelName(); def != "" {+ req.ModelName = def+ } else {+ return nil, fmt.Errorf("component: LLM: model_id is required and no default model is configured")+ }}driver := req.DrivermodelName := req.ModelNamediff --git a/internal/agent/harness/agentic_rag.go b/internal/agent/harness/agentic_rag.gonew file mode 100644index 0000000000..24b419199d--- /dev/null+++ b/internal/agent/harness/agentic_rag.go@@ -0,0 +1,107 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "log"+ "strings"++ "gorm.io/gorm"+)++// AgenticState carries the shared state across the agentic-RAG graph nodes.+type AgenticState struct {+ Question string+ Keywords string+ Route RouteDecision+ SeedChunks []string+ Plan WorkflowPlan+ Kbinfos *Kbinfos+ PartialAnswer bool+ Abstain bool+ EmptyResult bool+ FinalAnswer string+ FormalizeError string+}++// RunAgenticRAG drives the agentic-search graph: route → pre_search → planner →+// orchestrator → formalize_answer. Mirrors build_agentic_graph's linear flow.+//+// - low (direct_search): one hybrid search → answer.+// - medium+ (decompose_and_search / agentic_research / deep_research):+// pre_search grounds the planner, then decompose-and-search runs until a+// sufficiency verdict stops it.+func RunAgenticRAG(ctx context.Context, db *gorm.DB, question, keywords, modeLabel string, search SearchFn) AnswerResult {+ state := &AgenticState{+ Question: strings.TrimSpace(question),+ Keywords: keywords,+ Kbinfos: &Kbinfos{},+ }+ if state.Question == "" {+ return AnswerResult{FinalAnswer: emptyResultMessage, Empty: true}+ }++ // ── route ──+ state.Route = RouteNode(ctx, db, state.Question, modeLabel)++ // ── pre_search (decomposition modes only) ──+ if state.Route.RequiresDecomposition {+ chunks, aggs := search(ctx, state.Question, state.Keywords)+ state.SeedChunks = extractChunkTexts(chunks)+ state.Kbinfos.Merge(chunks, aggs)+ }++ // ── planner ──+ state.Plan = PlannerNode(ctx, db, state.Route, state.SeedChunks)++ // ── orchestrator ──+ var orch OrchestratorResult+ if state.Route.RequiresDecomposition {+ claims := make([]*ClaimTarget, len(state.Plan.Claims))+ for i := range state.Plan.Claims {+ claims[i] = &state.Plan.Claims[i]+ }+ orch = DecomposeAndSearch(ctx, search, state.Question, state.Keywords, claims, modeLabel, state.Kbinfos)+ } else {+ orch = DirectSearch(ctx, search, state.Question, state.Keywords, state.Kbinfos)+ }+ state.PartialAnswer = orch.PartialAnswer+ state.Abstain = orch.Abstain+ state.EmptyResult = orch.EmptyResult+ if orch.Kbinfos != nil {+ state.Kbinfos = orch.Kbinfos+ }++ // ── formalize_answer ──+ res := FormalizeAnswer(ctx, db, state.Question, state.Kbinfos, state.PartialAnswer, state.Abstain, state.EmptyResult)+ // Log only the question length, never its content, to avoid persisting user+ // input in logs.+ log.Printf("agentic_rag: finished (qlen=%d, strategy=%s, chunks=%d, partial=%v, abstain=%v)",+ len(state.Question), state.Route.ExecutionStrategy, len(state.Kbinfos.Chunks), state.PartialAnswer, state.Abstain)+ return res+}++func extractChunkTexts(chunks []map[string]interface{}) []string {+ out := make([]string, 0, len(chunks))+ for _, c := range chunks {+ if t := chunkText(c); t != "" {+ out = append(out, t)+ }+ }+ return out+}diff --git a/internal/agent/harness/answer.go b/internal/agent/harness/answer.gonew file mode 100644index 0000000000..6443cb7824--- /dev/null+++ b/internal/agent/harness/answer.go@@ -0,0 +1,124 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "fmt"+ "strings"++ "github.com/cloudwego/eino/schema"++ "gorm.io/gorm"+ "ragflow/internal/agent/chat"+)++// FINAL_ANSWER_SYSTEM mirrors Python report_prompt.FINAL_ANSWER_SYSTEM.+const finalAnswerSystem = `You are a smart agent. Answer the user's question using ONLY the evidence provided below. Do not invent facts: if the evidence cannot support a claim, say so plainly instead of guessing.++# Citation rules+{cite_rules}++# Language+Answer in the SAME language as the question. Translate retrieved evidence into that language as part of composing the answer; only verbatim quoted snippets may stay in their source language.++# Fallback+If the evidence does not answer the question, reply with a clear statement that you don't have enough information based on the available sources (in the user's language).+`++const partialAnswerPreamble = "Note: the following answer is based on partial information and may be incomplete."++const defaultCiteRules = "Cite passages with their source documents where available."++const (+ abstainMessage = "I cannot answer this question based on the available information."+ emptyResultMessage = "I don't have enough information based on the available sources."+)++// AnswerResult mirrors the formalize_answer node output.+type AnswerResult struct {+ FinalAnswer string+ Abstained bool+ Empty bool+}++// FormalizeAnswer generates the final answer from the gathered kbinfos. Mirrors+// Python's formalize_answer node: abstain/empty short-circuits, otherwise builds+// system+user and calls the chat invoker.+func FormalizeAnswer(ctx context.Context, db *gorm.DB, question string, kb *Kbinfos, partial, abstain, empty bool) AnswerResult {+ if abstain {+ return AnswerResult{FinalAnswer: abstainMessage, Abstained: true}+ }+ if empty || kb == nil || len(kb.Chunks) == 0 {+ return AnswerResult{FinalAnswer: emptyResultMessage, Empty: true}+ }++ var b strings.Builder+ b.WriteString("Question:\n" + question + "\n")+ if partial {+ b.WriteString(partialAnswerPreamble + "\n")+ }+ b.WriteString("\nEvidence:\n")+ for i, c := range kb.Chunks {+ text := chunkText(c)+ doc := chunkDoc(c)+ if text == "" {+ continue+ }+ b.WriteString(fmt.Sprintf("[%d] %s", i+1, text))+ if doc != "" {+ b.WriteString(fmt.Sprintf(" (source: %s)", doc))+ }+ b.WriteString("\n")+ }++ system := strings.ReplaceAll(finalAnswerSystem, "{cite_rules}", defaultCiteRules)+ inv := chat.GetDefaultInvoker()+ if inv == nil {+ return AnswerResult{FinalAnswer: "I'm sorry, the chat invoker is not configured."}+ }+ resp, err := inv.Invoke(ctx, db, chat.Request{+ Messages: []schema.Message{+ {Role: schema.System, Content: system},+ {Role: schema.User, Content: b.String()},+ },+ })+ if err != nil {+ return AnswerResult{FinalAnswer: "I'm sorry, I encountered an error while composing the answer."}+ }+ return AnswerResult{FinalAnswer: resp.Content}+}++func chunkText(c map[string]interface{}) string {+ if t, ok := c["content_with_weight"].(string); ok && t != "" {+ return t+ }+ if t, ok := c["content"].(string); ok {+ return t+ }+ return ""+}++func chunkDoc(c map[string]interface{}) string {+ if d, ok := c["docnm_kwd"].(string); ok {+ return d+ }+ if d, ok := c["doc_id"].(string); ok {+ return d+ }+ return ""+}diff --git a/internal/agent/harness/answer_test.go b/internal/agent/harness/answer_test.gonew file mode 100644index 0000000000..0520a8b6d5--- /dev/null+++ b/internal/agent/harness/answer_test.go@@ -0,0 +1,71 @@+package harness++import (+ "context"+ "strings"+ "testing"+)++// TestFormalizeAnswer_Abstain asserts abstain short-circuits to the abstain+// message without calling the model.+func TestFormalizeAnswer_Abstain(t *testing.T) {+ res := FormalizeAnswer(context.Background(), nil, "Q", &Kbinfos{}, false, true, false)+ if !res.Abstained || res.FinalAnswer != abstainMessage {+ t.Errorf("abstain result = %+v, want abstain message", res)+ }+}++// TestFormalizeAnswer_Empty asserts empty chunks short-circuit to the empty+// message.+func TestFormalizeAnswer_Empty(t *testing.T) {+ res := FormalizeAnswer(context.Background(), nil, "Q", &Kbinfos{}, false, false, false)+ if !res.Empty || res.FinalAnswer != emptyResultMessage {+ t.Errorf("empty result = %+v, want empty message", res)+ }+}++// TestFormalizeAnswer_Generates asserts a chat invoker is used to compose the+// final answer from the evidence.+func TestFormalizeAnswer_Generates(t *testing.T) {+ installChat(t, "here is the final answer")+ kb := &Kbinfos{Chunks: []map[string]interface{}{{"content_with_weight": "evidence alpha"}}}+ res := FormalizeAnswer(context.Background(), nil, "What is X?", kb, false, false, false)+ if res.FinalAnswer != "here is the final answer" {+ t.Errorf("final answer = %q, want chat output", res.FinalAnswer)+ }+}++// TestFormalizeAnswer_PartialPreamble asserts the partial preamble is included+// when partial=true.+func TestFormalizeAnswer_PartialPreamble(t *testing.T) {+ installChat(t, "partial ans")+ kb := &Kbinfos{Chunks: []map[string]interface{}{{"content_with_weight": "evidence"}}}+ res := FormalizeAnswer(context.Background(), nil, "Q", kb, true, false, false)+ if !strings.Contains(res.FinalAnswer, "partial ans") {+ t.Errorf("final answer = %q", res.FinalAnswer)+ }+}++// TestRunAgenticRAG_LowMode asserts low mode does a single direct search and+// produces an answer.+func TestRunAgenticRAG_LowMode(t *testing.T) {+ installChat(t, "final composed answer")+ res := RunAgenticRAG(context.Background(), nil, "What is a rocket?", "", "low",+ func(_ context.Context, _, _ string) ([]map[string]interface{}, []map[string]interface{}) {+ return []map[string]interface{}{{"chunk_id": "a", "content_with_weight": "rocket evidence"}}, nil+ })+ if res.FinalAnswer != "final composed answer" {+ t.Errorf("final answer = %q", res.FinalAnswer)+ }+}++// TestRunAgenticRAG_EmptySearch asserts an empty search yields the empty message.+func TestRunAgenticRAG_EmptySearch(t *testing.T) {+ res := RunAgenticRAG(context.Background(), nil, "Q", "", "low",+ func(_ context.Context, _, _ string) ([]map[string]interface{}, []map[string]interface{}) {+ return nil, nil+ })+ if !res.Empty || res.FinalAnswer != emptyResultMessage {+ t.Errorf("empty result = %+v", res)+ }+}diff --git a/internal/agent/harness/datasetnav.go b/internal/agent/harness/datasetnav.gonew file mode 100644index 0000000000..a82cf92d30--- /dev/null+++ b/internal/agent/harness/datasetnav.go@@ -0,0 +1,225 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "fmt"+ "log"+ "strings"++ "github.com/cloudwego/eino/schema"++ "gorm.io/gorm"+ "ragflow/internal/agent/chat"+ "ragflow/internal/service/nav"+)++// Dataset-nav router tunables mirror Python navigation.py.+const (+ navMaxDocs = 8+ navMaxClusters = 500+ navChildrenPageSize = 1000+ navTreeMaxDepth = 6+ navTreeMaxLeaves = 300+)++// navSelectSystem mirrors Python _NAV_SELECT_SYSTEM.+const navSelectSystem = `You are routing a question through a dataset's navigation tree.++You are given a QUESTION and a numbered list of {noun}, each with a name and a short description.+Choose the {noun} most likely to contain information relevant to answering the question.++Rules:+1. Judge only from the names and descriptions shown.+2. Be selective — include an item only if it is plausibly relevant. Include several when several are equally plausible.+3. If none are clearly relevant, return an empty list.+4. Return the bracketed index numbers of the chosen {noun}.++Output ONLY JSON, no prose, no code fences:+{"relevant": [<index>, ...]}`++type navSelectVerdict struct {+ Relevant []int `json:"relevant"`+}++// NavigateDatasetByTree walks the dataset nav tree with two LLM passes+// (cluster-select → document-select) and returns the routed doc_ids (capped at+// navMaxDocs). This is the LLM two-round selection (P1b) implemented in the+// harness package, which can import the chat invoker (agent/tool cannot without+// an import cycle). It routes only — it does not retrieve.+func NavigateDatasetByTree(ctx context.Context, db *gorm.DB, ns nav.NavService, tenantID, kbID, query string) []string {+ query = strings.TrimSpace(query)+ if query == "" {+ return nil+ }++ // 1. List top-level clusters.+ clusters, _, err := ns.ListClusters(ctx, tenantID, kbID, 0, navMaxClusters)+ if err != nil {+ log.Printf("datasetnav: list clusters failed: %v", err)+ return nil+ }+ if len(clusters) == 0 {+ return nil+ }+ clusterItems := make([]navSelectItem, len(clusters))+ for i, c := range clusters {+ clusterItems[i] = navSelectItem{Name: c.Name, Description: c.Description, DocCount: c.DocCount}+ }++ // 2. LLM selects relevant clusters.+ selected := askNavSelect(ctx, db, query, "clusters", clusterItems, navMaxClusters)+ if len(selected) == 0 {+ return nil+ }++ // 3. BFS-descend selected clusters to document leaves.+ leaves := collectNavLeaves(ctx, ns, tenantID, kbID, selected)++ // 4. LLM selects relevant documents.+ docs := askNavSelect(ctx, db, query, "documents", leaves, navTreeMaxLeaves)++ // Dedup + cap.+ seen := map[string]bool{}+ var routed []string+ for _, d := range docs {+ if d.DocID == "" || seen[d.DocID] {+ continue+ }+ seen[d.DocID] = true+ routed = append(routed, d.DocID)+ if len(routed) >= navMaxDocs {+ break+ }+ }+ return routed+}++// navSelectItem is a renderable cluster/document with a name + description.+type navSelectItem struct {+ Name string+ Description string+ DocCount int+ DocID string+}++// askNavSelect renders items as a numbered list and asks the model which indices+// are relevant. Returns the selected items (a subset). Mirrors _ask_nav_select.+func askNavSelect(ctx context.Context, db *gorm.DB, query, noun string, items []navSelectItem, maxItems int) []navSelectItem {+ if len(items) == 0 {+ return nil+ }+ capped := items+ if len(capped) > maxItems {+ capped = capped[:maxItems]+ }+ var b strings.Builder+ for i, it := range capped {+ name := strings.TrimSpace(it.Name)+ if name == "" {+ name = fmt.Sprintf("item-%d", i)+ }+ desc := strings.Join(strings.Fields(it.Description), " ")+ if len(desc) > 300 {+ desc = desc[:300]+ }+ b.WriteString(fmt.Sprintf("[%d] %s", i, name))+ if it.DocCount > 0 {+ b.WriteString(fmt.Sprintf(" [%d docs]", it.DocCount))+ }+ b.WriteString(": " + desc + "\n")+ }++ system := strings.ReplaceAll(navSelectSystem, "{noun}", noun)+ user := fmt.Sprintf("Question:\n%s\n\n%s (numbered):\n%s\n\nOutput JSON:", query, strings.Title(noun), b.String())++ inv := chat.GetDefaultInvoker()+ if inv == nil {+ log.Printf("datasetnav: LLM %s selection skipped (chat invoker not configured)", noun)+ return nil+ }+ resp, err := inv.Invoke(ctx, db, chat.Request{+ Messages: []schema.Message{+ {Role: schema.System, Content: system},+ {Role: schema.User, Content: user},+ },+ })+ if err != nil {+ log.Printf("datasetnav: LLM %s selection failed: %v", noun, err)+ return nil+ }+ var v navSelectVerdict+ if err := unmarshalModelJSON(resp.Content, &v); err != nil {+ return nil+ }+ seen := map[int]bool{}+ var out []navSelectItem+ for _, idx := range v.Relevant {+ if idx < 0 || idx >= len(capped) || seen[idx] {+ continue+ }+ seen[idx] = true+ out = append(out, capped[idx])+ }+ return out+}++// collectNavLeaves BFS-descents selected clusters to document leaves. Mirrors+// Python _collect_nav_leaves.+func collectNavLeaves(ctx context.Context, ns nav.NavService, tenantID, kbID string, selected []navSelectItem) []navSelectItem {+ type node struct {+ name string+ depth int+ }+ frontier := make([]node, 0, len(selected))+ for _, c := range selected {+ if c.Name != "" {+ frontier = append(frontier, node{c.Name, 0})+ }+ }+ var leaves []navSelectItem+ seenDocs := map[string]bool{}+ seenNodes := map[string]bool{}+ for len(frontier) > 0 && len(leaves) < navTreeMaxLeaves {+ cur := frontier[0]+ frontier = frontier[1:]+ if seenNodes[cur.name] {+ continue+ }+ seenNodes[cur.name] = true+ children, _, err := ns.ListChildren(ctx, tenantID, kbID, cur.name, 0, navChildrenPageSize)+ if err != nil {+ continue+ }+ for _, ch := range children {+ if ch.Type == "doc" {+ did := strings.TrimSpace(ch.DocID)+ if did != "" && !seenDocs[did] {+ seenDocs[did] = true+ leaves = append(leaves, navSelectItem{Name: ch.Name, Description: ch.Description, DocID: did})+ if len(leaves) >= navTreeMaxLeaves {+ break+ }+ }+ } else if ch.Type == "cluster" && ch.Name != "" && cur.depth+1 < navTreeMaxDepth {+ frontier = append(frontier, node{ch.Name, cur.depth + 1})+ }+ }+ }+ return leaves+}diff --git a/internal/agent/harness/datasetnav_test.go b/internal/agent/harness/datasetnav_test.gonew file mode 100644index 0000000000..a4b6e2cf0e--- /dev/null+++ b/internal/agent/harness/datasetnav_test.go@@ -0,0 +1,102 @@+package harness++import (+ "context"+ "testing"++ "ragflow/internal/service/nav"+)++// TestAskNavSelect_IndexBased asserts the model's index-based selection maps+// back to the item subset.+func TestAskNavSelect_IndexBased(t *testing.T) {+ installChat(t, `{"relevant":[0,2]}`)+ items := []navSelectItem{+ {Name: "Alpha", Description: "aaa"},+ {Name: "Beta", Description: "bbb"},+ {Name: "Gamma", Description: "ggg"},+ }+ out := askNavSelect(context.Background(), nil, "query", "clusters", items, 10)+ if len(out) != 2 {+ t.Fatalf("selected = %d, want 2", len(out))+ }+ if out[0].Name != "Alpha" || out[1].Name != "Gamma" {+ t.Errorf("selected names = %q, %q; want Alpha, Gamma", out[0].Name, out[1].Name)+ }+}++// TestAskNavSelect_Empty asserts an empty "relevant" list yields nothing.+func TestAskNavSelect_Empty(t *testing.T) {+ installChat(t, `{"relevant":[]}`)+ if out := askNavSelect(context.Background(), nil, "q", "clusters", []navSelectItem{{Name: "A"}}, 10); len(out) != 0 {+ t.Errorf("expected no selection, got %d", len(out))+ }+}++// TestAskNavSelect_OutOfRange asserts invalid indices are skipped.+func TestAskNavSelect_OutOfRange(t *testing.T) {+ installChat(t, `{"relevant":[0,99,-1]}`)+ out := askNavSelect(context.Background(), nil, "q", "clusters", []navSelectItem{{Name: "A"}, {Name: "B"}}, 10)+ if len(out) != 1 || out[0].Name != "A" {+ t.Errorf("out-of-range selection = %+v, want [A]", out)+ }+}++// TestNavigateDatasetByTree_NoQuery asserts empty query returns nil without+// calling anything.+func TestNavigateDatasetByTree_NoQuery(t *testing.T) {+ if out := NavigateDatasetByTree(context.Background(), nil, nil, "t1", "kb1", " "); out != nil {+ t.Errorf("expected nil for empty query, got %v", out)+ }+}++// fakeNavSvcHarness is an in-memory nav.NavService for the BFS test.+type fakeNavSvcHarness struct {+ clusters []nav.NavNode+ children map[string][]nav.NavNode+}++func (f *fakeNavSvcHarness) UpsertDoc(context.Context, nav.UpsertDocInput) error { return nil }+func (f *fakeNavSvcHarness) RemoveDoc(context.Context, string, string, string) error {+ return nil+}+func (f *fakeNavSvcHarness) Search(context.Context, string, string, string, []float32, int) ([]nav.NavHit, error) {+ return nil, nil+}+func (f *fakeNavSvcHarness) ListClusters(context.Context, string, string, int, int) ([]nav.NavNode, int64, error) {+ return f.clusters, int64(len(f.clusters)), nil+}+func (f *fakeNavSvcHarness) ListChildren(_ context.Context, _, _, name string, _, _ int) ([]nav.NavNode, int64, error) {+ return f.children[name], int64(len(f.children[name])), nil+}++// TestCollectNavLeaves_BFS asserts document leaves are collected, sub-clusters+// descended, and leaves deduped by doc_id.+func TestCollectNavLeaves_BFS(t *testing.T) {+ ns := &fakeNavSvcHarness{+ clusters: []nav.NavNode{{Name: "C1", Description: "cluster 1"}},+ children: map[string][]nav.NavNode{+ "C1": {+ {Name: "Sub", Type: "cluster"},+ {Name: "DocA", Type: "doc", DocID: "d1"},+ {Name: "DocB", Type: "doc", DocID: "d2"},+ },+ "Sub": {{Name: "DocC", Type: "doc", DocID: "d3"}},+ },+ }+ selected := []navSelectItem{{Name: "C1", Description: "cluster 1"}}+ leaves := collectNavLeaves(context.Background(), ns, "t1", "kb1", selected)+ if len(leaves) != 3 {+ t.Fatalf("leaves = %d, want 3 (d1,d2 from C1 + d3 from Sub)", len(leaves))+ }+ seen := map[string]bool{}+ for _, l := range leaves {+ if l.DocID == "" {+ t.Errorf("leaf %q has empty doc_id", l.Name)+ }+ seen[l.DocID] = true+ }+ if !seen["d1"] || !seen["d2"] || !seen["d3"] {+ t.Errorf("collected doc_ids = %v, want d1,d2,d3", seen)+ }+}diff --git a/internal/agent/harness/navigation.go b/internal/agent/harness/navigation.gonew file mode 100644index 0000000000..459f0c54ca--- /dev/null+++ b/internal/agent/harness/navigation.go@@ -0,0 +1,281 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "encoding/json"+ "fmt"+ "strings"++ "github.com/cloudwego/eino/schema"++ "ragflow/internal/agent/chat"+ "ragflow/internal/engine"+ "ragflow/internal/engine/types"+)++// Structure navigation mirrors Python navigation.py (ontology_navigate /+// mindmap_navigate). These live in the harness package (not agent/tool) because+// they need the chat invoker, and agent/tool → agent/component would form an+// import cycle.+const (+ toolOntologyNavigate = "ontology_navigate"+ toolMindmapNavigate = "mindmap_navigate"+)++var catalogKinds = map[string]bool{"tree": true, "timeline": true, "raptor": true, "page_index": true, "pageindex": true}+var mindmapKinds = map[string]bool{"mindmap": true, "mind_map": true}++const navSystemPrompt = `You are given the {noun} of one or more documents — an outline of entities and their relations — and a question.++Decide whether that outline alone already answers the question.++Rules:+1. Answer ONLY from the outline below. Do not invent facts.+2. Set "is_sufficient" to true only when the outline genuinely answers the question; otherwise false with an empty answer.+3. Always fill "relevant_entities" with the exact ` + "`name`" + ` values of the entities most related to the question (up to 10), even when the outline is not sufficient — they are used to pull the underlying source text.++Output ONLY JSON, no prose, no code fences:+{"is_sufficient": true/false, "answer": "<answer, or empty>", "relevant_entities": ["<entity name>", ...]}`++const (+ maxStructureEntities = 300+ maxStructureRelations = 300+ maxEvidenceChunks = 24+)++type structureNavArgs struct {+ Topic string `json:"topic"`+ Keywords string `json:"keywords,omitempty"`+ DocScope []string `json:"doc_scope,omitempty"`+}++type structureEntity struct {+ Name string `json:"name"`+ Type string `json:"type"`+ Description string `json:"description"`+ SourceChunkIDs []string `json:"source_chunk_ids"`+ DocID string `json:"-"`+}++type structureNavVerdict struct {+ IsSufficient bool `json:"is_sufficient"`+ Answer string `json:"answer"`+ RelevantEntities []string `json:"relevant_entities"`+}++// NavigateStructure implements ontology_navigate / mindmap_navigate. It reads+// the compiled structure (entities) of the in-scope documents, asks the chat+// model which entities answer the question, and pulls the source chunks behind+// the selected entities. Routing only — returns empty on any failure.+func NavigateStructure(ctx context.Context, tenantID string, kind string, args structureNavArgs) (string, error) {+ noun := "catalog"+ var kinds map[string]bool+ if kind == toolMindmapNavigate {+ kinds = mindmapKinds+ noun = "mindmap"+ } else {+ kinds = catalogKinds+ }++ query := strings.TrimSpace(args.Topic + " " + args.Keywords)+ if query == "" || len(args.DocScope) == 0 {+ return `{"chunks":[]}`, nil+ }++ var entities []structureEntity+ for _, docID := range args.DocScope {+ es := loadStructureEntities(ctx, tenantID, docID, kinds)+ for _, e := range es {+ if e.Name != "" {+ e.DocID = docID+ entities = append(entities, e)+ }+ }+ }+ if len(entities) == 0 {+ return `{"chunks":[]}`, nil+ }++ selected, err := askStructureSelect(ctx, query, noun, entities)+ if err != nil || len(selected) == 0 {+ return `{"chunks":[]}`, nil+ }++ idsByDoc := map[string][]string{}+ for _, e := range selected {+ idsByDoc[e.DocID] = append(idsByDoc[e.DocID], e.SourceChunkIDs...)+ }+ var chunks []map[string]interface{}+ for _, ids := range idsByDoc {+ chunks = append(chunks, loadChunksByIDs(ctx, tenantID, dedupStrings(ids))...)+ if len(chunks) >= maxEvidenceChunks {+ break+ }+ }+ b, _ := json.Marshal(map[string]interface{}{"chunks": chunks})+ return string(b), nil+}++func loadStructureEntities(ctx context.Context, tenantID, docID string, kinds map[string]bool) []structureEntity {+ de := engine.Get()+ if de == nil {+ return nil+ }+ idx := fmt.Sprintf("ragflow_%s", tenantID)+ req := &types.SearchRequest{+ IndexNames: []string{idx},+ Filter: map[string]interface{}{"doc_id": []string{docID}, "knowledge_graph_kwd": []string{"graph"}},+ SelectFields: []string{"content_with_weight", "compile_kwd", "compilation_template_kind_kwd"},+ Limit: 1000,+ }+ res, err := de.Search(ctx, req)+ if err != nil {+ return nil+ }+ var out []structureEntity+ for _, row := range res.Chunks {+ kind := normalizeKind(row)+ if !kinds[kind] {+ continue+ }+ payload, _ := row["content_with_weight"].(string)+ var graph struct {+ Entities []structureEntity `json:"entities"`+ }+ if err := json.Unmarshal([]byte(payload), &graph); err != nil {+ continue+ }+ out = append(out, graph.Entities...)+ }+ return out+}++func normalizeKind(row map[string]interface{}) string {+ if ck, _ := row["compile_kwd"].(string); ck == "raptor_graph" {+ return "raptor"+ }+ kind, _ := row["compilation_template_kind_kwd"].(string)+ if kind == "" {+ kind, _ = row["compile_kwd"].(string)+ }+ kind = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(kind, "-", "_")))+ if kind == "pageindex" || kind == "page_index" || kind == "knowledge_graph" {+ return "timeline"+ }+ return kind+}++func askStructureSelect(ctx context.Context, query, noun string, entities []structureEntity) ([]structureEntity, error) {+ rendered := renderStructureEntities(entities)+ inv := chat.GetDefaultInvoker()+ if inv == nil {+ return nil, fmt.Errorf("dataset navigation: chat invoker not configured")+ }+ resp, err := inv.Invoke(ctx, nil, chat.Request{+ Messages: []schema.Message{+ {Role: schema.System, Content: strings.ReplaceAll(navSystemPrompt, "{noun}", noun)},+ {Role: schema.User, Content: fmt.Sprintf("Question:\n%s\n\n%s:\n%s\n\nOutput JSON:", query, noun, rendered)},+ },+ })+ if err != nil {+ return nil, err+ }+ var v structureNavVerdict+ if err := unmarshalModelJSON(resp.Content, &v); err != nil {+ return nil, err+ }+ want := map[string]bool{}+ for _, n := range v.RelevantEntities {+ want[n] = true+ }+ var out []structureEntity+ for _, e := range entities {+ if want[e.Name] {+ out = append(out, e)+ }+ }+ return out, nil+}++func renderStructureEntities(entities []structureEntity) string {+ var b strings.Builder+ b.WriteString("Entities:")+ for i, e := range entities {+ if i >= maxStructureEntities {+ break+ }+ b.WriteString("\n- " + e.Name + " (" + orStr(e.Type, "other") + ")")+ if d := strings.Join(strings.Fields(e.Description), " "); d != "" {+ b.WriteString(": " + d)+ }+ }+ return b.String()+}++func loadChunksByIDs(ctx context.Context, tenantID string, ids []string) []map[string]interface{} {+ if len(ids) == 0 {+ return nil+ }+ de := engine.Get()+ if de == nil {+ return nil+ }+ idx := fmt.Sprintf("ragflow_%s", tenantID)+ limit := maxEvidenceChunks+ if len(ids) < limit {+ limit = len(ids)+ }+ req := &types.SearchRequest{+ IndexNames: []string{idx},+ Filter: map[string]interface{}{"id": ids},+ SelectFields: []string{"content_with_weight", "docnm_kwd", "doc_id"},+ Limit: limit,+ }+ res, err := de.Search(ctx, req)+ if err != nil {+ return nil+ }+ var out []map[string]interface{}+ for _, row := range res.Chunks {+ out = append(out, map[string]interface{}{+ "chunk_id": row["id"], "content_with_weight": row["content_with_weight"],+ "docnm_kwd": row["docnm_kwd"], "doc_id": row["doc_id"],+ })+ }+ return out+}++func dedupStrings(in []string) []string {+ seen := map[string]bool{}+ var out []string+ for _, s := range in {+ if s != "" && !seen[s] {+ seen[s] = true+ out = append(out, s)+ }+ }+ return out+}++func orStr(v, def string) string {+ if v == "" {+ return def+ }+ return v+}diff --git a/internal/agent/harness/navigation_test.go b/internal/agent/harness/navigation_test.gonew file mode 100644index 0000000000..e566378390--- /dev/null+++ b/internal/agent/harness/navigation_test.go@@ -0,0 +1,65 @@+package harness++import (+ "context"+ "strings"+ "testing"+)++// TestRenderStructureEntities asserts the prompt outline is compact and includes+// entity name/type/description.+func TestRenderStructureEntities(t *testing.T) {+ entities := []structureEntity{+ {Name: "Rocket", Type: "concept", Description: "a propulsion device"},+ {Name: "Engine", Type: "part"},+ }+ out := renderStructureEntities(entities)+ if out == "" {+ t.Fatal("expected rendered outline")+ }+ if !containsAll(out, "Rocket", "Engine", "concept") {+ t.Errorf("rendered outline missing entities: %q", out)+ }+}++// TestNavigateStructure_NoScope asserts empty doc scope yields empty chunks+// without calling the model.+func TestNavigateStructure_NoScope(t *testing.T) {+ out, err := NavigateStructure(context.Background(), "t1", toolOntologyNavigate, structureNavArgs{+ Topic: "rocket", DocScope: nil,+ })+ if err != nil {+ t.Fatal(err)+ }+ if out != `{"chunks":[]}` {+ t.Errorf("no-scope result = %q, want empty chunks", out)+ }+}++// TestNormalizeKind mirrors the API's kind normalization.+func TestNormalizeKind(t *testing.T) {+ cases := []struct {+ row map[string]interface{}+ want string+ }{+ {map[string]interface{}{"compile_kwd": "raptor_graph"}, "raptor"},+ {map[string]interface{}{"compilation_template_kind_kwd": "page_index"}, "timeline"},+ {map[string]interface{}{"compilation_template_kind_kwd": "knowledge_graph"}, "timeline"},+ {map[string]interface{}{"compilation_template_kind_kwd": "mindmap"}, "mindmap"},+ {map[string]interface{}{"compile_kwd": "tree"}, "tree"},+ }+ for _, c := range cases {+ if got := normalizeKind(c.row); got != c.want {+ t.Errorf("normalizeKind(%v) = %q, want %q", c.row, got, c.want)+ }+ }+}++func containsAll(s string, subs ...string) bool {+ for _, sub := range subs {+ if !strings.Contains(s, sub) {+ return false+ }+ }+ return true+}diff --git a/internal/agent/harness/orchestrator.go b/internal/agent/harness/orchestrator.gonew file mode 100644index 0000000000..7c9cbc8b1c--- /dev/null+++ b/internal/agent/harness/orchestrator.go@@ -0,0 +1,247 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "fmt"+ "strings"+)++// Orchestration input/output mirrors Python orchestrator (direct.py / decompose.py).+// SearchFn performs one hybrid search for a query and returns chunks + doc aggs,+// so the orchestrator is decoupled from the concrete retrieval backend.+type SearchFn func(ctx context.Context, query, keywords string) ([]map[string]interface{}, []map[string]interface{})++// Kbinfos is the shared accumulation store (Python tools.kbinfos).+type Kbinfos struct {+ Chunks []map[string]interface{}+ DocAggs []map[string]interface{}+}++func (k *Kbinfos) HasChunks() bool { return len(k.Chunks) > 0 }++// Merge appends the given chunks/aggs, deduplicating by chunkKey. It returns the+// GLOBAL indices (positions in k.Chunks after the merge) of the chunks this call+// contributed, so callers can store stable evidence references across multiple+// Merge calls (per-search indices would diverge from the accumulated list).+func (k *Kbinfos) Merge(chunks, aggs []map[string]interface{}) []int {+ seen := map[string]bool{}+ for _, c := range k.Chunks {+ seen[chunkKey(c)] = true+ }+ var added []int+ for _, c := range chunks {+ kk := chunkKey(c)+ if !seen[kk] {+ seen[kk] = true+ k.Chunks = append(k.Chunks, c)+ }+ // Record the global index of every contributed chunk (dedup or new),+ // so EvidenceIDs always reference the accumulated kbinfos positions.+ added = append(added, indexOfChunk(k.Chunks, kk))+ }+ dseen := map[string]bool{}+ for _, d := range k.DocAggs {+ if id, _ := d["doc_id"].(string); id != "" {+ dseen[id] = true+ }+ }+ for _, d := range aggs {+ if id, _ := d["doc_id"].(string); id != "" && !dseen[id] {+ dseen[id] = true+ k.DocAggs = append(k.DocAggs, d)+ }+ }+ return added+}++// indexOfChunk returns the global index of the chunk whose key matches kk.+func indexOfChunk(chunks []map[string]interface{}, kk string) int {+ for i, c := range chunks {+ if chunkKey(c) == kk {+ return i+ }+ }+ return -1+}++// chunkKey returns a stable dedup key for a chunk. Prefers chunk_id/id when+// present; otherwise falls back to a content-derived hash so chunks without an+// id still dedup correctly (and never collide on a shared empty/"" key).+func chunkKey(c map[string]interface{}) string {+ if id, ok := c["chunk_id"].(string); ok && id != "" {+ return "cid:" + id+ }+ if id, ok := c["id"].(string); ok && id != "" {+ return "id:" + id+ }+ content := ""+ if t, ok := c["content_with_weight"].(string); ok {+ content = t+ } else if t, ok := c["content"].(string); ok {+ content = t+ }+ if content == "" {+ // No stable identity: fall back to the doc reference so at least+ // per-document grouping is preserved (rarely reached).+ content = fmt.Sprintf("%s|%s", anyString(c["doc_id"]), anyString(c["docnm_kwd"]))+ }+ return "h:" + fnv64(content)+}++func anyString(v interface{}) string {+ if s, ok := v.(string); ok {+ return s+ }+ return ""+}++// fnv64 is a deterministic non-crypto hash for dedup keys.+func fnv64(s string) string {+ h := uint64(14695981039346656037)+ for i := 0; i < len(s); i++ {+ h ^= uint64(s[i])+ h *= 1099511628211+ }+ return fmt.Sprintf("%016x", h)+}++// OrchestratorResult mirrors the state updates returned by the Python+// orchestrator nodes.+type OrchestratorResult struct {+ Verdict *SufficiencyVerdict+ PartialAnswer bool+ Abstain bool+ EmptyResult bool+ Kbinfos *Kbinfos+}++// DirectSearch is the low-mode orchestrator: one hybrid search → merge.+func DirectSearch(ctx context.Context, search SearchFn, question, keywords string, kbinfos *Kbinfos) OrchestratorResult {+ if kbinfos == nil {+ kbinfos = &Kbinfos{}+ }+ chunks, aggs := search(ctx, question, keywords)+ kbinfos.Merge(chunks, aggs)+ if !kbinfos.HasChunks() {+ return OrchestratorResult{EmptyResult: true, Kbinfos: kbinfos}+ }+ return OrchestratorResult{Kbinfos: kbinfos}+}++// DecomposeAndSearch is the medium-mode orchestrator: decompose → parallel+// search → cross-check → fusion → iterate until a verdict stops it.+func DecomposeAndSearch(ctx context.Context, search SearchFn, question, keywords string, claims []*ClaimTarget, modeLabel string, kbinfos *Kbinfos) OrchestratorResult {+ if kbinfos == nil {+ kbinfos = &Kbinfos{}+ }+ mode, _ := GetMode(modeLabel)+ if mode.Label == "" {+ mode = THINKING_MODES["medium"]+ }+ unverified := func() []*ClaimTarget {+ var out []*ClaimTarget+ for _, c := range claims {+ if !c.IsVerified {+ out = append(out, c)+ }+ }+ return out+ }++ for cycle := 0; cycle < mode.MaxOrchestratorCycles; cycle++ {+ uv := unverified()+ if len(uv) == 0 {+ break+ }+ for _, c := range uv {+ chunks, aggs := search(ctx, c.Description, keywords)+ if len(chunks) > 0 {+ c.IsVerified = true+ c.Confidence = 0.8+ // Merge returns the GLOBAL indices of this claim's chunks in the+ // accumulated kbinfos, which is what CrossCheckClaim resolves+ // against (allChunks is keyed by that global position).+ global := kbinfos.Merge(chunks, aggs)+ c.AgentResult = &AgentResult{+ ClaimID: c.ClaimID, Report: summarize(chunks), IsVerified: true, Confidence: 0.8,+ EvidenceIDs: global,+ }+ } else {+ c.AgentResult = &AgentResult{ClaimID: c.ClaimID, IsVerified: false, Confidence: 0.0}+ }+ }++ allChunks := map[int]map[string]interface{}{}+ for i, c := range kbinfos.Chunks {+ allChunks[i] = c+ }+ var agentResults []AgentResult+ var crossResults []ClaimCrossCheckResult+ for _, c := range claims {+ if c.AgentResult != nil {+ agentResults = append(agentResults, *c.AgentResult)+ crossResults = append(crossResults, CrossCheckClaim(c.AgentResult, allChunks))+ }+ }+ verdict := ComputeFusionScore(agentResults, crossResults, mode)+ action, _ := RouteSufficiencyVerdict(verdict, modeLabel, cycle, mode.MaxOrchestratorCycles)++ switch action {+ case "ANSWER", "ANSWER_PARTIAL":+ return OrchestratorResult{Verdict: &verdict, PartialAnswer: action == "ANSWER_PARTIAL", Kbinfos: kbinfos}+ case "ABSTAIN":+ kbinfos.Chunks = nil+ return OrchestratorResult{Verdict: &verdict, Abstain: true, Kbinfos: kbinfos}+ case "REPLAN":+ // Reset unverified for another cycle (simplified: continue loop).+ case "CONTINUE":+ // fallthrough to next cycle+ }+ }+ return OrchestratorResult{Kbinfos: kbinfos}+}++func summarize(chunks []map[string]interface{}) string {+ var parts []string+ n := 3+ if len(chunks) < n {+ n = len(chunks)+ }+ for _, c := range chunks[:n] {+ text := ""+ if t, ok := c["content_with_weight"].(string); ok {+ text = t+ } else if t, ok := c["content"].(string); ok {+ text = t+ }+ if len(text) > 200 {+ text = text[:200]+ }+ parts = append(parts, text)+ }+ return strings.Join(parts, " | ")+}++func intRange(n int) []int {+ out := make([]int, n)+ for i := range out {+ out[i] = i+ }+ return out+}diff --git a/internal/agent/harness/orchestrator_test.go b/internal/agent/harness/orchestrator_test.gonew file mode 100644index 0000000000..daefd5d8d0--- /dev/null+++ b/internal/agent/harness/orchestrator_test.go@@ -0,0 +1,135 @@+package harness++import (+ "context"+ "testing"+)++// TestCrossCheckClaim_NumberMatch asserts a verified report whose numbers appear+// in the evidence chunk passes the cross-check.+func TestCrossCheckClaim_NumberMatch(t *testing.T) {+ agent := &AgentResult{ClaimID: "c0", IsVerified: true, Report: "speed is 88 and 12", EvidenceIDs: []int{0}}+ chunks := map[int]map[string]interface{}{0: {"content_with_weight": "speed is 88 and 12 here"}}+ r := CrossCheckClaim(agent, chunks)+ if !r.CrossCheckPassed {+ t.Errorf("cross-check should pass, got mismatches %v", r.Mismatches)+ }+}++// TestKbinfosMerge_GlobalIndices asserts Merge returns the GLOBAL indices of the+// contributed chunks in the accumulated list, so a later claim's EvidenceIDs+// still resolve correctly after earlier claims pushed more chunks in.+func TestKbinfosMerge_GlobalIndices(t *testing.T) {+ kb := &Kbinfos{}+ // First claim's search returns chunks a,b.+ first := kb.Merge([]map[string]interface{}{+ {"chunk_id": "a", "content_with_weight": "alpha 7"},+ {"chunk_id": "b", "content_with_weight": "beta"},+ }, nil)+ // a->0, b->1 in the global list.+ if len(first) != 2 || first[0] != 0 || first[1] != 1 {+ t.Fatalf("first merge global indices = %v, want [0 1]", first)+ }+ // Second claim's search returns chunk c (now global index 2) and a dup of a.+ second := kb.Merge([]map[string]interface{}{+ {"chunk_id": "c", "content_with_weight": "gamma 9"},+ {"chunk_id": "a", "content_with_weight": "alpha 7"},+ }, nil)+ // c->2, a(dup)->0; NOT per-search [0 1].+ if len(second) != 2 || second[0] != 2 || second[1] != 0 {+ t.Fatalf("second merge global indices = %v, want [2 0]", second)+ }+ // CrossCheckClaim must resolve both claims' evidence against the global list.+ allChunks := map[int]map[string]interface{}{}+ for i, c := range kb.Chunks {+ allChunks[i] = c+ }+ r1 := CrossCheckClaim(&AgentResult{ClaimID: "c1", IsVerified: true, Report: "alpha 7", EvidenceIDs: first}, allChunks)+ if !r1.HasEvidence {+ t.Error("claim 1 should have evidence at global indices")+ }+ r2 := CrossCheckClaim(&AgentResult{ClaimID: "c2", IsVerified: true, Report: "gamma 9", EvidenceIDs: second}, allChunks)+ if !r2.HasEvidence {+ t.Error("claim 2 should have evidence at global index 2 (not per-search 0)")+ }+}++// TestCrossCheckClaim_Unverified asserts an unverified agent fails the check.+func TestCrossCheckClaim_Unverified(t *testing.T) {+ r := CrossCheckClaim(&AgentResult{ClaimID: "c0", IsVerified: false}, nil)+ if r.CrossCheckPassed {+ t.Error("unverified agent must fail cross-check")+ }+}++// TestComputeFusionScore_Sufficient asserts a fully-verified high-score set is+// SUFFICIENT.+func TestComputeFusionScore_Sufficient(t *testing.T) {+ agents := []AgentResult{{ClaimID: "c0", IsVerified: true, Report: "value 42", EvidenceIDs: []int{0}}}+ cross := []ClaimCrossCheckResult{{ClaimID: "c0", CrossCheckPassed: true, CrossCheckScore: 1.0, HasEvidence: true}}+ v := ComputeFusionScore(agents, cross, THINKING_MODES["medium"])+ if v.Status != "SUFFICIENT" {+ t.Errorf("status = %q, want SUFFICIENT", v.Status)+ }+}++// TestComputeFusionScore_NoEvidence asserts a claim with no examined evidence is+// UNANSWERABLE (empty-evidence guard).+func TestComputeFusionScore_NoEvidence(t *testing.T) {+ cross := []ClaimCrossCheckResult{{ClaimID: "c0", CrossCheckPassed: true, CrossCheckScore: 1.0, HasEvidence: false}}+ v := ComputeFusionScore([]AgentResult{{ClaimID: "c0", IsVerified: true}}, cross, THINKING_MODES["medium"])+ if v.Status != "UNANSWERABLE" {+ t.Errorf("status = %q, want UNANSWERABLE", v.Status)+ }+ if len(v.MissingClaims) != 1 {+ t.Errorf("missing claims = %v, want [c0]", v.MissingClaims)+ }+}++// TestRouteSufficiencyVerdict asserts SUFFICIENT → ANSWER.+func TestRouteSufficiencyVerdict(t *testing.T) {+ action, cont := RouteSufficiencyVerdict(SufficiencyVerdict{Status: "SUFFICIENT", Score: 0.9}, "medium", 0, 3)+ if action != "ANSWER" || cont {+ t.Errorf("got (%q,%v), want (ANSWER,false)", action, cont)+ }+}++// TestDirectSearch_Merges asserts direct search merges chunks and flags empty.+func TestDirectSearch_Merges(t *testing.T) {+ kb := &Kbinfos{}+ res := DirectSearch(context.Background(), func(_ context.Context, _, _ string) ([]map[string]interface{}, []map[string]interface{}) {+ return []map[string]interface{}{{"chunk_id": "a", "content_with_weight": "alpha"}}, nil+ }, "q", "", kb)+ if res.EmptyResult || !kb.HasChunks() {+ t.Errorf("expected merged chunks, empty=%v chunks=%d", res.EmptyResult, len(kb.Chunks))+ }+}++// TestDirectSearch_Empty asserts direct search flags empty when no chunks.+func TestDirectSearch_Empty(t *testing.T) {+ kb := &Kbinfos{}+ res := DirectSearch(context.Background(), func(_ context.Context, _, _ string) ([]map[string]interface{}, []map[string]interface{}) {+ return nil, nil+ }, "q", "", kb)+ if !res.EmptyResult {+ t.Error("expected empty_result=true")+ }+}++// TestDecomposeAndSearch_Verifies asserts a searchable claim gets verified and+// the loop stops on ANSWER.+func TestDecomposeAndSearch_Verifies(t *testing.T) {+ claims := []*ClaimTarget{{ClaimID: "c0", Description: "fact 42 about X"}}+ kb := &Kbinfos{}+ res := DecomposeAndSearch(context.Background(),+ func(_ context.Context, _, _ string) ([]map[string]interface{}, []map[string]interface{}) {+ return []map[string]interface{}{{"chunk_id": "x", "content_with_weight": "the value is 42"}}, nil+ },+ "Q", "", claims, "medium", kb)+ if !claims[0].IsVerified {+ t.Error("claim should be verified")+ }+ if res.Verdict == nil {+ t.Error("expected a verdict")+ }+}diff --git a/internal/agent/harness/planner.go b/internal/agent/harness/planner.gonew file mode 100644index 0000000000..a7203e39ec--- /dev/null+++ b/internal/agent/harness/planner.go@@ -0,0 +1,164 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "fmt"+ "log"+ "strings"++ "github.com/cloudwego/eino/schema"++ "ragflow/internal/agent/chat"++ "gorm.io/gorm"+)++// decomposePrompt is a faithful port of Python's DECOMPOSE_FACTUAL prompt shape.+// Question-type-specific variants share the same structure but tune the+// instructions; the Go planner selects by question_type.+const decomposePrompt = `Break down the research question into %d atomic claims (facts or sub-questions).++Question: %s++Detail level: %s+Grounding context (preliminary retrieval, may be empty):+%s++Output format:+{+ "claims": [+ {"claim_id": "c0", "description": "...", "priority": 0, "suggested_tools": []}+ ]+}+`++type plannerClaim struct {+ ClaimID string `json:"claim_id"`+ Description string `json:"description"`+ Priority int `json:"priority"`+ SuggestedTools []string `json:"suggested_tools"`+}++type plannerResult struct {+ Claims []plannerClaim `json:"claims"`+}++// PlannerNode mirrors Python planner_node. It decomposes the routed question+// into ClaimTargets. Direct mode (no decomposition) returns a single coarse+// claim. On any failure it falls back to the direct plan.+func PlannerNode(ctx context.Context, db *gorm.DB, route RouteDecision, seedChunks []string) WorkflowPlan {+ if !route.RequiresDecomposition {+ return directPlan(route.Question)+ }+ mode, ok := GetMode(route.ThinkingMode)+ if !ok {+ // Unknown or empty mode label: fall back to medium so the planner is not+ // driven by a zero-valued mode (which would yield a degenerate plan).+ mode = THINKING_MODES["medium"]+ }+ maxClaims := maxClaimsFor(mode.Label)+ detail := detailLevelFor(mode.Label)++ prompt := fmt.Sprintf(decomposePrompt, maxClaims, route.Question, detail, formatSeedChunks(seedChunks))++ inv := chat.GetDefaultInvoker()+ if inv == nil {+ log.Printf("agentic_rag: planner_node skipped (chat invoker not configured); fallback direct")+ return directPlan(route.Question)+ }+ resp, err := inv.Invoke(ctx, db, chat.Request{+ Messages: []schema.Message{+ {Role: schema.System, Content: prompt},+ {Role: schema.User, Content: route.Question},+ },+ })+ if err != nil {+ log.Printf("agentic_rag: planner_node failed (fallback direct): %v", err)+ return directPlan(route.Question)+ }+ var res plannerResult+ if err := unmarshalModelJSON(resp.Content, &res); err != nil {+ log.Printf("agentic_rag: planner_node parse failed (fallback direct): %v", err)+ return directPlan(route.Question)+ }+ claims := make([]ClaimTarget, 0, len(res.Claims))+ for i, c := range res.Claims {+ desc := strings.TrimSpace(c.Description)+ if desc == "" {+ continue+ }+ claims = append(claims, ClaimTarget{+ ClaimID: orDefault(c.ClaimID, fmt.Sprintf("c%d", i)),+ Description: desc,+ Priority: c.Priority,+ SuggestedTools: c.SuggestedTools,+ })+ }+ if len(claims) == 0 {+ return directPlan(route.Question)+ }+ return WorkflowPlan{+ PlanType: planTypeFor(route.QuestionType),+ Claims: claims,+ MaxIterations: mode.MaxOrchestratorCycles,+ }+}++func directPlan(question string) WorkflowPlan {+ return WorkflowPlan{+ PlanType: "direct",+ Claims: []ClaimTarget{{ClaimID: "c0", Description: question, Priority: 0}},+ MaxIterations: 1,+ }+}++func planTypeFor(qt string) string {+ switch qt {+ case "factual":+ return "fact_decomposition"+ case "comparative":+ return "comparative_decomposition"+ case "procedural":+ return "procedural_decomposition"+ default:+ return "exploratory_decomposition"+ }+}++func maxClaimsFor(modeLabel string) int {+ return map[string]int{"low": 1, "medium": 3, "high": 5, "ultra": 8}[modeLabel]+}++func detailLevelFor(modeLabel string) string {+ return map[string]string{"low": "coarse", "medium": "normal", "high": "fine", "ultra": "extra_fine"}[modeLabel]+}++func formatSeedChunks(seedChunks []string) string {+ if len(seedChunks) == 0 {+ return "(no preliminary results)"+ }+ return strings.Join(seedChunks, "\n")+}++func orDefault(v, def string) string {+ if v == "" {+ return def+ }+ return v+}diff --git a/internal/agent/harness/production.go b/internal/agent/harness/production.gonew file mode 100644index 0000000000..2a6f3a9139--- /dev/null+++ b/internal/agent/harness/production.go@@ -0,0 +1,147 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "encoding/json"+ "fmt"+ "log"+ "strings"++ einotool "github.com/cloudwego/eino/components/tool"++ "gorm.io/gorm"+ "ragflow/internal/agent/tool"+ "ragflow/internal/service/nav"+)++// ProductionRunner wires the real agentic-search tools (hybrid_search,+// dataset_navigation_by_tree) into the RunAgenticRAG flow, so the tools are+// actually invoked rather than merely registered. This is the production+// counterpart to the unit-testable SearchFn seam.+type ProductionRunner struct {+ db *gorm.DB+ tenantID string+ datasetIDs []string+ searchTool einotool.InvokableTool+ navSvc nav.NavService // defaults to nav.GetNavService() when nil+}++// NewProductionRunner builds a ProductionRunner backed by the real tools. The+// dataset-nav router (harness.NavigateDatasetByTree) resolves its NavService+// lazily via nav.GetNavService().+func NewProductionRunner(db *gorm.DB, tenantID string, datasetIDs []string) (*ProductionRunner, error) {+ searchBase, err := tool.BuildByName("hybrid_search", nil)+ if err != nil {+ return nil, err+ }+ search, ok := searchBase.(einotool.InvokableTool)+ if !ok {+ return nil, fmt.Errorf("hybrid_search is not invokable")+ }+ return &ProductionRunner{db: db, tenantID: tenantID, datasetIDs: datasetIDs, searchTool: search}, nil+}++// newProductionRunnerWithTools builds a ProductionRunner with an injected+// search tool and nav service, for unit/E2E tests that want to fake the+// invocation surface without real services.+func newProductionRunnerWithTools(db *gorm.DB, tenantID string, datasetIDs []string, searchTool einotool.InvokableTool, navSvc nav.NavService) *ProductionRunner {+ return &ProductionRunner{db: db, tenantID: tenantID, datasetIDs: datasetIDs, searchTool: searchTool, navSvc: navSvc}+}++// Run executes the agentic-search graph with the real tools. It returns the+// final answer.+func (r *ProductionRunner) Run(ctx context.Context, question, keywords, modeLabel string) AnswerResult {+ if r.searchTool == nil {+ log.Printf("agentic_rag: production runner not fully wired (search tool missing)")+ return AnswerResult{FinalAnswer: emptyResultMessage, Empty: true}+ }+ // The router tool returns a doc list; feed it as the search DocScope.+ searchFn := func(ctx context.Context, query, kws string) ([]map[string]interface{}, []map[string]interface{}) {+ return r.search(ctx, query, kws, nil)+ }++ // For decomposition modes, route the doc scope first via the nav tool.+ if mode, _ := GetMode(modeLabel); mode.RequiresDecomposition {+ docs := r.routeDocs(ctx, question, keywords)+ if len(docs) > 0 {+ searchFn = func(ctx context.Context, query, kws string) ([]map[string]interface{}, []map[string]interface{}) {+ return r.search(ctx, query, kws, docs)+ }+ }+ }+ return RunAgenticRAG(ctx, r.db, question, keywords, modeLabel, searchFn)+}++// search invokes the hybrid_search tool and normalizes its chunk output.+func (r *ProductionRunner) search(ctx context.Context, query, keywords string, docScope []string) ([]map[string]interface{}, []map[string]interface{}) {+ args := map[string]interface{}{"query": query, "keywords": keywords, "kb_ids": r.datasetIDs, "top_n": 12}+ if len(docScope) > 0 {+ args["doc_scope"] = docScope+ }+ raw, err := r.searchTool.InvokableRun(ctx, mustJSON(args))+ if err != nil {+ log.Printf("agentic_rag: hybrid_search failed: %v", err)+ return nil, nil+ }+ var res struct {+ Chunks []map[string]interface{} `json:"chunks"`+ }+ if err := json.Unmarshal([]byte(raw), &res); err != nil {+ return nil, nil+ }+ return res.Chunks, nil+}++// routeDocs derives the doc scope via the canonical dataset-nav router+// (harness.NavigateDatasetByTree — the full LLM two-round selection). It routes+// across ALL bound datasets and merges the doc ids, so every KB contributes its+// own relevant docs to the shared scope (a multi-KB session must not collapse to+// the first KB only).+func (r *ProductionRunner) routeDocs(ctx context.Context, topic, keywords string) []string {+ ns := r.navSvc+ if ns == nil {+ ns = nav.GetNavService()+ }+ if ns == nil {+ log.Printf("agentic_rag: dataset nav service not initialized; skipping doc routing")+ return nil+ }+ // Combine topic + keywords into the routing query so the nav router actually+ // uses the full user signal (keywords must not be dropped).+ query := strings.TrimSpace(topic + " " + keywords)+ seen := map[string]bool{}+ var docs []string+ for _, kbID := range r.datasetIDs {+ for _, id := range NavigateDatasetByTree(ctx, r.db, ns, r.tenantID, kbID, query) {+ if id != "" && !seen[id] {+ seen[id] = true+ docs = append(docs, id)+ }+ }+ }+ return docs+}++func mustJSON(v interface{}) string {+ b, err := json.Marshal(v)+ if err != nil {+ return "{}"+ }+ return string(b)+}diff --git a/internal/agent/harness/production_test.go b/internal/agent/harness/production_test.gonew file mode 100644index 0000000000..cd53998d99--- /dev/null+++ b/internal/agent/harness/production_test.go@@ -0,0 +1,144 @@+package harness++import (+ "context"+ "strings"+ "sync"+ "testing"++ einotool "github.com/cloudwego/eino/components/tool"+ "github.com/cloudwego/eino/schema"+ "gorm.io/gorm"++ "ragflow/internal/agent/component"+ "ragflow/internal/service/nav"+)++// routeChat returns a model-style response depending on the stage: nav-selection+// calls return {"relevant":[...]}, all other calls return a plain final answer.+type routeChat struct{}++func (routeChat) Invoke(_ context.Context, _ *gorm.DB, req component.ChatInvokeRequest) (*component.ChatInvokeResponse, error) {+ content := ""+ msg := ""+ if len(req.Messages) > 0 {+ msg = req.Messages[len(req.Messages)-1].Content+ }+ switch {+ case strings.Contains(msg, "(numbered)") || strings.Contains(msg, "Entities"):+ // nav-select pass: keep every item (all relevant).+ content = `{"relevant":[0,1,2,3,4,5,6,7,8,9]}`+ default:+ content = "final scoped answer"+ }+ return &component.ChatInvokeResponse{Content: content}, nil+}++// installRouteChat installs the stage-aware chat invoker for E2E tests.+func installRouteChat(t *testing.T) {+ t.Helper()+ component.SetDefaultChatInvoker(routeChat{})+ t.Cleanup(func() { component.SetDefaultChatInvoker(nil) })+}++// fakeInvokableTool is an einotool.InvokableTool double for E2E testing.+type fakeInvokableTool struct {+ name string+ fn func(ctx context.Context, argsJSON string) string+ mu sync.Mutex+ lastArgs string+}++func (f *fakeInvokableTool) Info(_ context.Context) (*schema.ToolInfo, error) {+ return &schema.ToolInfo{Name: f.name}, nil+}++func (f *fakeInvokableTool) InvokableRun(_ context.Context, argsJSON string, _ ...einotool.Option) (string, error) {+ f.mu.Lock()+ f.lastArgs = argsJSON+ f.mu.Unlock()+ return f.fn(context.Background(), argsJSON), nil+}++func (f *fakeInvokableTool) args() string {+ f.mu.Lock()+ defer f.mu.Unlock()+ return f.lastArgs+}++// TestProductionRunner_E2E_RouteToScopedSearch proves the chain+// "user question → dataset nav routes doc scope → hybrid_search retrieval is+// scoped to those docs". It injects a fake nav service (returns docs d1,d2) and+// asserts the search tool receives doc_scope=[d1,d2].+func TestProductionRunner_E2E_RouteToScopedSearch(t *testing.T) {+ installRouteChat(t)++ // Fake nav service whose two-round router returns docs d1,d2.+ navSvc := &fakeNavSvcHarness{+ clusters: []nav.NavNode{{Name: "C1", Description: "cluster"}},+ children: map[string][]nav.NavNode{+ "C1": {+ {Name: "DocA", Type: "doc", DocID: "d1"},+ {Name: "DocB", Type: "doc", DocID: "d2"},+ },+ },+ }+ // AskNavSelect returns all items (both docs) for the doc-select pass.+ searchTool := &fakeInvokableTool{name: "hybrid_search", fn: func(_ context.Context, _ string) string {+ return `{"chunks":[{"chunk_id":"c1","content_with_weight":"scoped evidence"}]}`+ }}++ // Multi-KB session: routing must cover every bound dataset, not just the+ // first KB.+ runner := newProductionRunnerWithTools(nil, "t1", []string{"kb1", "kb2"}, searchTool, navSvc)+ res := runner.Run(context.Background(), "Compare A and B", "", "medium")++ if res.FinalAnswer != "final scoped answer" {+ t.Errorf("final answer = %q, want chat output", res.FinalAnswer)+ }+ // The search tool must receive doc_scope=[d1,d2] — the docs the nav router+ // selected — proving retrieval is scoped to those docs across KBs.+ if !strings.Contains(searchTool.args(), `"doc_scope":["d1","d2"]`) {+ t.Errorf("hybrid_search not scoped to routed docs; args=%s", searchTool.args())+ }+ // The search tool must receive all bound KBs (not collapsed to one).+ if !strings.Contains(searchTool.args(), `"kb_ids":["kb1","kb2"]`) {+ t.Errorf("hybrid_search kb_ids must include all bound KBs; args=%s", searchTool.args())+ }+}++// TestProductionRunner_E2E_NoRoute_SearchUnscoped asserts low mode (no+// decomposition) skips nav routing and searches without a doc_scope.+func TestProductionRunner_E2E_NoRoute_SearchUnscoped(t *testing.T) {+ installRouteChat(t)+ searchTool := &fakeInvokableTool{name: "hybrid_search", fn: func(_ context.Context, _ string) string {+ return `{"chunks":[{"chunk_id":"c1","content_with_weight":"evidence"}]}`+ }}+ // A nav service is provided but must NOT be consulted in low mode.+ navSvc := &fakeNavSvcHarness{clusters: []nav.NavNode{{Name: "C1"}}}+ runner := newProductionRunnerWithTools(nil, "t1", []string{"kb1"}, searchTool, navSvc)+ res := runner.Run(context.Background(), "What is X?", "", "low")++ if res.FinalAnswer == "" {+ t.Error("expected a final answer in low mode")+ }+ if strings.Contains(searchTool.args(), "doc_scope") {+ t.Errorf("low mode search should have no doc_scope; args=%s", searchTool.args())+ }+}++// TestProductionRunner_E2E_EmptyRoute_SearchUnscoped asserts an empty doc route+// falls back to an unscoped search (no hard failure).+func TestProductionRunner_E2E_EmptyRoute_SearchUnscoped(t *testing.T) {+ installRouteChat(t)+ searchTool := &fakeInvokableTool{name: "hybrid_search", fn: func(_ context.Context, _ string) string {+ return `{"chunks":[{"chunk_id":"c1","content_with_weight":"evidence"}]}`+ }}+ // Nav service with no clusters -> empty route.+ navSvc := &fakeNavSvcHarness{clusters: nil}+ runner := newProductionRunnerWithTools(nil, "t1", []string{"kb1"}, searchTool, navSvc)+ res := runner.Run(context.Background(), "Compare A and B", "", "medium")+ if res.FinalAnswer == "" {+ t.Error("expected an answer even when nav routing returns no docs")+ }+}diff --git a/internal/agent/harness/route.go b/internal/agent/harness/route.gonew file mode 100644index 0000000000..5459cfd0d5--- /dev/null+++ b/internal/agent/harness/route.go@@ -0,0 +1,133 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "encoding/json"+ "log"+ "regexp"+ "strings"++ "github.com/cloudwego/eino/schema"++ "ragflow/internal/agent/chat"++ "gorm.io/gorm"+)++// routePrompt mirrors Python harness/prompts/route_prompt.py.+const routePrompt = `Analyze the following question and output a structured query analysis.++Question: %s++Analyze it across these dimensions:+1. Question type: factual / comparative / analytical / procedural / exploratory / verification / summarization.+2. Whether it needs decomposition into atomic facts, meaning whether multiple independent pieces of information must be retrieved separately before answering: true/false.+3. Suggested knowledge compilation tool: null (none) / toc (document table of contents) / graph (knowledge graph) / wiki (compiled domain knowledge).++Output format (JSON):+{+ "question_type": "comparative",+ "requires_decomposition": true,+ "suggests_compilation": null,+ "reasoning": "This is a comparative question, so it needs to be decomposed into two independent facts and one comparison relation."+}+`++type routeResult struct {+ QuestionType string `json:"question_type"`+ RequiresDecomp *bool `json:"requires_decomposition"`+ Reasoning string `json:"reasoning"`+}++// RouteNode mirrors Python route_node. It classifies the question into a+// RouteDecision using the default chat invoker. Pure classification, no KB+// dependency. Never fails — falls back to a factual/direct decision.+func RouteNode(ctx context.Context, db *gorm.DB, question, modeLabel string) RouteDecision {+ if strings.TrimSpace(question) == "" {+ return fallbackRoute(question, modeLabel, "fallback: empty question")+ }+ inv := chat.GetDefaultInvoker()+ if inv == nil {+ return fallbackRoute(question, modeLabel, "fallback: chat invoker not configured")+ }+ resp, err := inv.Invoke(ctx, db, chat.Request{+ Messages: []schema.Message{+ {Role: schema.System, Content: strings.ReplaceAll(routePrompt, "%s", question)},+ {Role: schema.User, Content: question},+ },+ })+ if err != nil {+ log.Printf("agentic_rag: route_node failed (fallback): %v", err)+ return fallbackRoute(question, modeLabel, "fallback: LLM error")+ }+ var res routeResult+ if err := unmarshalModelJSON(resp.Content, &res); err != nil {+ log.Printf("agentic_rag: route_node parse failed (fallback): %v", err)+ return fallbackRoute(question, modeLabel, "fallback: parse error")+ }+ return decide(question, modeLabel, res)+}++// decide applies the mode's execution strategy to the LLM route output.+func decide(question, modeLabel string, res routeResult) RouteDecision {+ mode, ok := GetMode(modeLabel)+ if !ok {+ mode = THINKING_MODES["medium"]+ }+ qType := res.QuestionType+ if qType == "" {+ qType = "factual"+ }+ needDecomp := true+ if res.RequiresDecomp != nil {+ needDecomp = *res.RequiresDecomp+ }+ return RouteDecision{+ Question: question,+ ThinkingMode: modeLabel,+ QuestionType: qType,+ RequiresDecomposition: mode.RequiresDecomposition && needDecomp,+ ExecutionStrategy: mode.Strategy,+ Reasoning: res.Reasoning,+ }+}++func fallbackRoute(question, modeLabel, reason string) RouteDecision {+ return RouteDecision{+ Question: question, ThinkingMode: modeLabel, QuestionType: "factual",+ RequiresDecomposition: false, ExecutionStrategy: "direct_search", Reasoning: reason,+ }+}++var (+ reThinkTag = regexp.MustCompile(`(?s)^.*</think>`)+ reFence = regexp.MustCompile("```(?:json)?\\s*|\\s*```")+)++// unmarshalModelJSON mirrors Python's _extract_json: strip thinking preamble and+// markdown fences, then parse JSON.+func unmarshalModelJSON(text string, out interface{}) error {+ text = reThinkTag.ReplaceAllString(text, "")+ text = reFence.ReplaceAllString(text, "")+ text = strings.TrimSpace(text)+ if text == "" {+ return json.Unmarshal([]byte("{}"), out)+ }+ return json.Unmarshal([]byte(text), out)+}diff --git a/internal/agent/harness/route_test.go b/internal/agent/harness/route_test.gonew file mode 100644index 0000000000..03d9170079--- /dev/null+++ b/internal/agent/harness/route_test.go@@ -0,0 +1,140 @@+package harness++import (+ "context"+ "testing"++ "gorm.io/gorm"++ "ragflow/internal/agent/component"+)++// fakeChatInvoker returns a fixed content for the chat call, so tests can drive+// the route/planner LLM output deterministically.+type fakeChatInvoker struct{ content string }++func (f *fakeChatInvoker) Invoke(_ context.Context, _ *gorm.DB, _ component.ChatInvokeRequest) (*component.ChatInvokeResponse, error) {+ return &component.ChatInvokeResponse{Content: f.content}, nil+}++func installChat(t *testing.T, content string) {+ t.Helper()+ component.SetDefaultChatInvoker(&fakeChatInvoker{content: content})+ t.Cleanup(func() { component.SetDefaultChatInvoker(nil) })+}++// TestRouteNode_Classifies asserts route classification drives the execution+// strategy from the mode and the LLM's question_type.+func TestRouteNode_Classifies(t *testing.T) {+ installChat(t, `{"question_type":"comparative","requires_decomposition":true,"reasoning":"cmp"}`)+ r := RouteNode(context.Background(), nil, "Compare A and B", "medium")+ if r.QuestionType != "comparative" {+ t.Errorf("question_type = %q, want comparative", r.QuestionType)+ }+ // medium mode requires decomposition AND LLM says true -> true.+ if !r.RequiresDecomposition {+ t.Errorf("requires_decomposition = false, want true")+ }+ if r.ExecutionStrategy != "decompose_and_search" {+ t.Errorf("execution_strategy = %q, want decompose_and_search", r.ExecutionStrategy)+ }+}++// TestRouteNode_LowModeDisablesDecomposition asserts low mode never decomposes+// even if the LLM requests it.+func TestRouteNode_LowModeDisablesDecomposition(t *testing.T) {+ installChat(t, `{"question_type":"analytical","requires_decomposition":true}`)+ r := RouteNode(context.Background(), nil, "Analyze X", "low")+ if r.RequiresDecomposition {+ t.Errorf("low mode must disable decomposition, got true")+ }+ if r.ExecutionStrategy != "direct_search" {+ t.Errorf("low mode execution_strategy = %q, want direct_search", r.ExecutionStrategy)+ }+}++// TestRouteNode_FencedJSON asserts think-tag/fence stripping works.+func TestRouteNode_FencedJSON(t *testing.T) {+ installChat(t, "Sure!\n```json\n{\"question_type\":\"factual\",\"requires_decomposition\":false}\n```")+ r := RouteNode(context.Background(), nil, "What is X?", "medium")+ if r.QuestionType != "factual" {+ t.Errorf("question_type = %q, want factual", r.QuestionType)+ }+ if r.RequiresDecomposition {+ t.Errorf("requires_decomposition = true, want false")+ }+}++// TestRouteNode_EmptyQuestionFallsBack asserts an empty question yields a+// direct factual decision without calling the LLM.+func TestRouteNode_EmptyQuestionFallsBack(t *testing.T) {+ r := RouteNode(context.Background(), nil, "", "medium")+ if r.QuestionType != "factual" || r.RequiresDecomposition {+ t.Errorf("empty question must fall back to direct factual, got %+v", r)+ }+}++// TestPlannerNode_DirectMode asserts a non-decomposed route yields one coarse+// claim without calling the LLM.+func TestPlannerNode_DirectMode(t *testing.T) {+ plan := PlannerNode(context.Background(), nil, RouteDecision{+ Question: "What is X?", RequiresDecomposition: false,+ }, nil)+ if plan.PlanType != "direct" || len(plan.Claims) != 1 {+ t.Fatalf("direct plan = %+v, want 1 direct claim", plan)+ }+}++// TestPlannerNode_Decomposes asserts the planner builds claims from the LLM+// output and applies the mode's max iterations.+func TestPlannerNode_Decomposes(t *testing.T) {+ installChat(t, `{"claims":[+ {"claim_id":"c0","description":"fact one","priority":0},+ {"claim_id":"c1","description":"fact two","priority":1}+ ]}`)+ plan := PlannerNode(context.Background(), nil, RouteDecision{+ Question: "Compare A and B", QuestionType: "comparative", RequiresDecomposition: true, ThinkingMode: "medium",+ }, nil)+ if plan.PlanType != "comparative_decomposition" {+ t.Errorf("plan_type = %q, want comparative_decomposition", plan.PlanType)+ }+ if len(plan.Claims) != 2 {+ t.Fatalf("claims = %d, want 2", len(plan.Claims))+ }+ if plan.Claims[1].Priority != 1 {+ t.Errorf("claim priority = %d, want 1", plan.Claims[1].Priority)+ }+ // medium maxOrchestratorCycles = 3+ if plan.MaxIterations != 3 {+ t.Errorf("max_iterations = %d, want 3", plan.MaxIterations)+ }+}++// TestPlannerNode_UnknownModeFallsBack asserts an unknown (non-empty) mode label+// falls back to medium so the planner is not driven by a zero-valued mode+// (which would produce a degenerate plan with max_claims=0).+func TestPlannerNode_UnknownModeFallsBack(t *testing.T) {+ installChat(t, `{"claims":[{"claim_id":"c0","description":"fact one","priority":0}]}`)+ plan := PlannerNode(context.Background(), nil, RouteDecision{+ Question: "Q", RequiresDecomposition: true, ThinkingMode: "turbo-unknown",+ }, nil)+ // medium maxOrchestratorCycles = 3, and claims must still be built.+ if plan.MaxIterations != 3 {+ t.Errorf("max_iterations = %d, want 3 (medium fallback)", plan.MaxIterations)+ }+ if len(plan.Claims) != 1 {+ t.Errorf("claims = %d, want 1", len(plan.Claims))+ }+}++// TestPlannerNode_BadJSONFallsBack asserts unparseable planner output falls back+// to the direct plan.+func TestPlannerNode_BadJSONFallsBack(t *testing.T) {+ installChat(t, "not json at all")+ plan := PlannerNode(context.Background(), nil, RouteDecision{+ Question: "Q", RequiresDecomposition: true, ThinkingMode: "medium",+ }, nil)+ if plan.PlanType != "direct" || len(plan.Claims) != 1 {+ t.Fatalf("fallback plan = %+v, want direct", plan)+ }+}diff --git a/internal/agent/harness/sufficiency.go b/internal/agent/harness/sufficiency.gonew file mode 100644index 0000000000..beb4290133--- /dev/null+++ b/internal/agent/harness/sufficiency.go@@ -0,0 +1,250 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "fmt"+ "regexp"+ "strings"+)++// Sufficiency scoring (code-only, mirrors Python sufficiency.py): cross-check an+// agent result against the evidence chunks, fuse agent confidence + cross-check+// pass rate, then route to a 5-way verdict.++var reNumber = regexp.MustCompile(`\d+\.?\d*`)+var reEntities = regexp.MustCompile(`\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b`)++// extractNumbers returns numeric values found in text.+func extractNumbers(text string) []string {+ return reNumber.FindAllString(text, -1)+}++// extractNamedEntities returns capitalized multi-word sequences.+func extractNamedEntities(text string) []string {+ seen := map[string]bool{}+ var out []string+ for _, e := range reEntities.FindAllString(text, -1) {+ if !seen[e] {+ seen[e] = true+ out = append(out, e)+ }+ }+ return out+}++// CrossCheckClaim performs a code-level cross-check of an agent result against+// the accumulated evidence chunks (number matching + entity presence).+func CrossCheckClaim(agent *AgentResult, allChunks map[int]map[string]interface{}) ClaimCrossCheckResult {+ if agent == nil {+ return ClaimCrossCheckResult{ClaimID: "", CrossCheckPassed: false, Mismatches: []string{"nil agent result"}}+ }+ if !agent.IsVerified {+ return ClaimCrossCheckResult{ClaimID: agent.ClaimID, CrossCheckPassed: false, Mismatches: []string{"agent self-reported as unverified"}}+ }+ numbers := extractNumbers(agent.Report)+ entities := extractNamedEntities(agent.Report)++ var matches, mismatches []string+ for _, eid := range agent.EvidenceIDs {+ chunk, ok := allChunks[eid]+ if !ok {+ mismatches = append(mismatches, fmt.Sprintf("evidence_id=%d: chunk not found", eid))+ continue+ }+ text := ""+ if c, ok := chunk["content_with_weight"].(string); ok {+ text = strings.ToLower(c)+ } else if c, ok := chunk["content"].(string); ok {+ text = strings.ToLower(c)+ }+ for _, num := range numbers {+ if strings.Contains(text, num) {+ matches = append(matches, fmt.Sprintf("number %s found in chunk %d", num, eid))+ } else {+ mismatches = append(mismatches, fmt.Sprintf("number %s not found in chunk %d", num, eid))+ }+ }+ for _, ent := range entities {+ if strings.Contains(text, strings.ToLower(ent)) {+ matches = append(matches, fmt.Sprintf("entity '%s' found in chunk %d", ent, eid))+ } else {+ mismatches = append(mismatches, fmt.Sprintf("entity '%s' not found in chunk %d", ent, eid))+ }+ }+ }+ // HasEvidence is true when at least one evidence id resolved to a chunk with+ // content. A claim with no resolvable evidence is never considered verified.+ hasEvidence := len(matches)+len(mismatches) > 0+ total := len(matches) + len(mismatches)+ crossScore := 0.0+ if total > 0 {+ crossScore = float64(len(matches)) / float64(total)+ }+ // Entity presence now contributes to matches too, so it can raise the score;+ // use a float comparison to avoid integer-division truncation bias.+ crossPassed := hasEvidence && float64(len(mismatches)) < float64(len(matches))/2.0+ return ClaimCrossCheckResult{+ ClaimID: agent.ClaimID, CrossCheckPassed: crossPassed, CrossCheckScore: crossScore,+ EvidenceMatches: matches, Mismatches: mismatches, HasEvidence: hasEvidence,+ }+}++// ComputeFusionScore fuses agent confidence + cross-check pass rate into a+// SufficiencyVerdict for the given mode.+func ComputeFusionScore(agentResults []AgentResult, crossResults []ClaimCrossCheckResult, mode ExecutionStrategy) SufficiencyVerdict {+ verified := 0+ for _, r := range agentResults {+ if r.IsVerified {+ verified+++ }+ }+ agentScore := 0.0+ if len(agentResults) > 0 {+ agentScore = float64(verified) / float64(len(agentResults))+ }+ passed := 0+ for _, r := range crossResults {+ if r.CrossCheckPassed {+ passed+++ }+ }+ crossScore := 0.0+ if len(crossResults) > 0 {+ crossScore = float64(passed) / float64(len(crossResults))+ }++ fusionScore := agentScore+ if crossScore > fusionScore {+ fusionScore = crossScore // low/medium default: max+ }+ switch mode.Label {+ case "ultra":+ fusionScore = min(agentScore, crossScore)+ case "high":+ fusionScore = (agentScore + crossScore) / 2+ }++ hasConflicts := false+ for _, r := range crossResults {+ if len(r.Mismatches) > 0 {+ hasConflicts = true+ break+ }+ }++ // Empty-evidence guard: if no claim examined any evidence chunk, the answer+ // cannot be grounded at all — this is UNANSWERABLE, not merely incomplete.+ anyEvidence := false+ for _, r := range crossResults {+ if r.HasEvidence {+ anyEvidence = true+ break+ }+ }++ status := "INSUFFICIENT"+ switch {+ case !anyEvidence:+ status = "UNANSWERABLE"+ case hasConflicts && fusionScore < mode.PartialThreshold:+ status = "CONFLICTING"+ case fusionScore >= mode.SufficiencyThreshold:+ status = "SUFFICIENT"+ case fusionScore >= mode.PartialThreshold:+ status = "USEFUL_BUT_INCOMPLETE"+ case func() bool {+ for _, r := range crossResults {+ if r.CrossCheckPassed {+ return true+ }+ }+ return false+ }():+ status = "INSUFFICIENT"+ default:+ status = "UNANSWERABLE"+ }++ var missing []string+ for _, r := range crossResults {+ if !r.CrossCheckPassed || !r.HasEvidence {+ missing = append(missing, r.ClaimID)+ }+ }++ assessments := make([]map[string]interface{}, 0, len(crossResults))+ for _, r := range crossResults {+ assessments = append(assessments, map[string]interface{}{+ "claim_id": r.ClaimID, "is_verified": r.CrossCheckPassed && r.HasEvidence, "score": r.CrossCheckScore,+ "mismatches": r.Mismatches, "has_evidence": r.HasEvidence,+ })+ }++ return SufficiencyVerdict{+ Status: status, Score: fusionScore, AgentScore: agentScore, CrossScore: crossScore,+ ClaimAssessments: assessments, HasConflicts: hasConflicts, MissingClaims: missing,+ Feedback: buildFeedback(missing, crossResults), OverallReason: fmt.Sprintf("%s score=%.2f missing=%v", status, fusionScore, missing),+ }+}++func buildFeedback(missing []string, results []ClaimCrossCheckResult) string {+ if len(missing) == 0 {+ return "all claims verified"+ }+ var hints []string+ for _, r := range results {+ if !r.CrossCheckPassed {+ hints = append(hints, fmt.Sprintf("claim %s: %d mismatch(es)", r.ClaimID, len(r.Mismatches)))+ }+ }+ return "missing: " + strings.Join(hints, "; ")+}++// RouteSufficiencyVerdict returns (action, shouldContinue) from the verdict.+func RouteSufficiencyVerdict(v SufficiencyVerdict, modeLabel string, cycle, maxCycles int) (string, bool) {+ mode, _ := GetMode(modeLabel)+ if mode.Label == "" {+ mode = THINKING_MODES["medium"]+ }+ switch v.Status {+ case "SUFFICIENT":+ return "ANSWER", false+ case "USEFUL_BUT_INCOMPLETE":+ if mode.RequiresSelectiveGen {+ return "ANSWER_PARTIAL", false+ }+ return "CONTINUE", false+ case "INSUFFICIENT":+ if cycle >= int(float64(maxCycles)*0.8) {+ return "ANSWER_PARTIAL", false+ }+ return "CONTINUE", true+ case "CONFLICTING":+ if mode.AllowsReplan && cycle < int(float64(maxCycles)*0.5) {+ return "REPLAN", true+ }+ return "ANSWER_PARTIAL", false+ case "UNANSWERABLE":+ if mode.FallbackToDirectLLM {+ return "FALLBACK_LLM", false+ }+ return "ABSTAIN", false+ default:+ return "CONTINUE", true+ }+}diff --git a/internal/agent/harness/types.go b/internal/agent/harness/types.gonew file mode 100644index 0000000000..3b9c56d357--- /dev/null+++ b/internal/agent/harness/types.go@@ -0,0 +1,151 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// Package harness ports Python's rag/advanced_rag/harness (agentic-RAG):+// route → planner → orchestrator → formalize_answer. This is a phased port; see+// tasks/agentic_search_port_plan.md §P4.+package harness++// RouteDecision mirrors Python RouteDecision.+type RouteDecision struct {+ Question string+ ThinkingMode string+ QuestionType string // factual | comparative | analytical | procedural | exploratory | verification | summarization+ RequiresDecomposition bool+ ExecutionStrategy string // direct_search | decompose_and_search | agentic_research | deep_research+ Reasoning string+}++// ClaimTarget mirrors Python ClaimTarget.+type ClaimTarget struct {+ ClaimID string+ Description string+ Priority int+ SuggestedTools []string+ IsVerified bool+ Confidence float64+ AgentResult *AgentResult+}++// WorkflowPlan mirrors Python WorkflowPlan.+type WorkflowPlan struct {+ PlanType string // direct | fact_decomposition | ...+ Claims []ClaimTarget+ MaxIterations int+}++// ExecutionStrategy mirrors Python ExecutionStrategy and is instantiated by the+// four thinking modes.+type ExecutionStrategy struct {+ Label string+ Strategy string+ RequiresDecomposition bool+ MaxOrchestratorCycles int+ MaxAgentCycles int+ MaxParallelAgents int+ AvailableTools []string+ SufficiencyThreshold float64+ PartialThreshold float64+ FallbackToDirectLLM bool+ RequiresSelectiveGen bool+ AllowsReplan bool+}++// AgentResult mirrors Python AgentResult.+type AgentResult struct {+ ClaimID string+ Report string+ IsVerified bool+ Confidence float64+ EvidenceIDs []int+}++// ClaimCrossCheckResult mirrors Python ClaimCrossCheckResult.+type ClaimCrossCheckResult struct {+ ClaimID string+ CrossCheckPassed bool+ CrossCheckScore float64+ EvidenceMatches []string+ Mismatches []string+ // HasEvidence reports whether any evidence chunk was actually examined. When+ // false the claim had no resolvable evidence, so it cannot be considered+ // verified regardless of the other fields.+ HasEvidence bool+}++// SufficiencyVerdict mirrors Python SufficiencyVerdict.+type SufficiencyVerdict struct {+ Status string // SUFFICIENT | USEFUL_BUT_INCOMPLETE | INSUFFICIENT | CONFLICTING | UNANSWERABLE+ Score float64+ AgentScore float64+ CrossScore float64+ ClaimAssessments []map[string]interface{}+ HasConflicts bool+ MissingClaims []string+ Feedback string+ OverallReason string+}++// OrchestratorContext mirrors Python OrchestratorContext.+type OrchestratorContext struct {+ Question string+ Claims []*ClaimTarget+ Mode string+ Iteration int+}++// THINKING_MODES mirrors Python config.THINKING_MODES.+var THINKING_MODES = map[string]ExecutionStrategy{+ "low": {+ Label: "low", Strategy: "direct_search", RequiresDecomposition: false,+ MaxOrchestratorCycles: 1, MaxAgentCycles: 0, MaxParallelAgents: 1,+ AvailableTools: []string{"hybrid_search"}, SufficiencyThreshold: 0.85, PartialThreshold: 0.50,+ },+ "medium": {+ Label: "medium", Strategy: "decompose_and_search", RequiresDecomposition: true,+ MaxOrchestratorCycles: 3, MaxAgentCycles: 0, MaxParallelAgents: 1,+ AvailableTools: []string{"hybrid_search"}, SufficiencyThreshold: 0.75, PartialThreshold: 0.40,+ RequiresSelectiveGen: true,+ },+ "high": {+ Label: "high", Strategy: "agentic_research", RequiresDecomposition: true,+ MaxOrchestratorCycles: 3, MaxAgentCycles: 2, MaxParallelAgents: 2,+ AvailableTools: []string{+ "hybrid_search", "web_search", "ontology_navigate", "dataset_navigation_by_tree",+ "graph_explore", "inspector_open_context", "inspector_compare",+ },+ SufficiencyThreshold: 0.65, PartialThreshold: 0.30,+ RequiresSelectiveGen: true, AllowsReplan: true,+ },+ "ultra": {+ Label: "ultra", Strategy: "deep_research", RequiresDecomposition: true,+ MaxOrchestratorCycles: 4, MaxAgentCycles: 2, MaxParallelAgents: 3,+ AvailableTools: []string{+ "hybrid_search", "bm25_search", "web_search", "structured_query",+ "ontology_navigate", "dataset_navigation_by_tree", "mindmap_navigate",+ "graph_explore", "wiki_query", "inspector_open_context", "inspector_compare",+ "inspector_grep_within", "inspector_request_adjacent",+ },+ SufficiencyThreshold: 0.55, PartialThreshold: 0.20, FallbackToDirectLLM: true,+ RequiresSelectiveGen: true, AllowsReplan: true,+ },+}++// GetMode mirrors Python get_mode(label).+func GetMode(label string) (ExecutionStrategy, bool) {+ m, ok := THINKING_MODES[label]+ return m, ok+}diff --git a/internal/agent/tool/agentic_search.go b/internal/agent/tool/agentic_search.gonew file mode 100644index 0000000000..3a5de007f9--- /dev/null+++ b/internal/agent/tool/agentic_search.go@@ -0,0 +1,345 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package tool++import (+ "context"+ "encoding/json"+ "fmt"+ "regexp"+ "strings"++ einotool "github.com/cloudwego/eino/components/tool"+ "github.com/cloudwego/eino/schema"+)++// Agentic search tool names mirror Python rag/advanced_rag/harness/tools/search.py.+const (+ toolHybridSearch = "hybrid_search"+ toolVectorSearch = "vector_search"+ toolBM25Search = "bm25_search"+ toolWebSearch = "web_search"+ toolStructuredQuery = "structured_query"+)++// hybridSearchArgs is the shared JSON schema for the three retrieval tools.+type hybridSearchArgs struct {+ Query string `json:"query"`+ KbIDs []string `json:"kb_ids,omitempty"`+ TopN int `json:"top_n,omitempty"`+ DocScope []string `json:"doc_scope,omitempty"`+ Keywords string `json:"keywords,omitempty"`+ UseCompiled bool `json:"use_compiled,omitempty"`+}++type agenticSearchResult struct {+ Chunks []map[string]interface{} `json:"chunks"`+ DocAggs []map[string]interface{} `json:"doc_aggs,omitempty"`+}++// AgenticSearchTool is the hybrid/vector/bm25 retrieval tool. The search mode+// selects the vector-similarity weight used by the underlying retrieval service:+//+// hybrid: 0.3 (hybrid of keyword + vector)+// vector: 1.0 (vector-only)+// bm25: 0.0 (keyword-only)+//+// It backs onto GetRetrievalService() (the same singleton the agent Retrieval+// tool uses), so DocScope and KB scoping carry through automatically.+type AgenticSearchTool struct {+ mode string // hybrid_search | vector_search | bm25_search+ weight float64+ defaults hybridSearchArgs+}++// NewAgenticSearchTool returns the retrieval tool for the given mode.+func NewAgenticSearchTool(mode string) *AgenticSearchTool {+ weight := 0.3+ switch mode {+ case toolVectorSearch:+ weight = 1.0+ case toolBM25Search:+ weight = 0.0+ }+ return &AgenticSearchTool{mode: mode, weight: weight, defaults: hybridSearchArgs{TopN: 12}}+}++func (a *AgenticSearchTool) Info(_ context.Context) (*schema.ToolInfo, error) {+ return &schema.ToolInfo{+ Name: a.mode,+ Desc: fmt.Sprintf("Search the bound knowledge base(s) for the query (mode=%s). Returns relevant passages.", a.mode),+ ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{+ "query": {+ Type: schema.String, Required: true, Desc: "The search query.",+ },+ "kb_ids": {Type: schema.Array, Desc: "Optional dataset ids to restrict to."},+ "top_n": {Type: schema.Number, Desc: "Number of passages to return (default 12)."},+ "doc_scope": {Type: schema.Array, Desc: "Optional doc ids to restrict to."},+ "keywords": {Type: schema.String, Desc: "Comma-separated keywords to narrow results."},+ "use_compiled": {Type: schema.Boolean, Desc: "Whether to enrich with compiled products."},+ }),+ }, nil+}++// InvokableRun executes the retrieval. It returns JSON with "chunks" (array of+// chunk maps). Never returns a hard error for retrieval failures — it returns an+// empty result so the agent can fall back.+func (a *AgenticSearchTool) InvokableRun(ctx context.Context, argumentsInJSON string, _ ...einotool.Option) (string, error) {+ var args hybridSearchArgs+ if err := json.Unmarshal([]byte(argumentsInJSON), &args); err != nil {+ return "", fmt.Errorf("%s: parse arguments: %w", a.mode, err)+ }+ if args.TopN <= 0 {+ args.TopN = a.defaults.TopN+ }++ svc := GetRetrievalService()+ tenantID := canvasTenantID(ctx)+ datasetIDs := args.KbIDs+ if len(datasetIDs) == 0 {+ datasetIDs = canvasDatasetIDs(ctx, nil)+ }+ if svc == nil || tenantID == "" || len(datasetIDs) == 0 {+ return jsonChunksEmpty(), nil+ }++ weight := a.weight+ req := RetrievalRequest{+ Query: strings.TrimSpace(args.Query + " " + args.Keywords),+ DatasetIDs: datasetIDs,+ TopN: args.TopN,+ TopK: args.TopN * 4,+ SimilarityThreshold: 0.2,+ KeywordsSimilarityWeight: &weight,+ DocScope: args.DocScope,+ }+ chunks, err := svc.Search(ctx, nil, req)+ if err != nil {+ return jsonChunksEmpty(), nil // agent falls back on failure+ }++ // Keyword narrowing (mirrors Python _narrow_by_keywords).+ if args.Keywords != "" {+ chunks = narrowByKeywords(chunks, args.Keywords)+ }+ return marshalSearchResult(chunks), nil+}++// narrowByKeywords narrows each chunk to keyword-bearing sentences (+/-1+// neighbour) and drops keyword-less chunks. A simplified port of Python's+// _narrow_by_keywords.+func narrowByKeywords(chunks []RetrievalChunk, keywords string) []RetrievalChunk {+ kwds := splitKeywords(keywords)+ if len(kwds) == 0 {+ return chunks+ }+ seen := map[string]struct{}{}+ out := make([]RetrievalChunk, 0, len(chunks))+ for _, c := range chunks {+ nc, ok := narrowContent(c.Content, kwds)+ if !ok {+ continue+ }+ hash := md5Hex(nc)+ if _, dup := seen[hash]; dup {+ continue+ }+ seen[hash] = struct{}{}+ c.Content = nc+ out = append(out, c)+ }+ return out+}++// splitKeywords normalizes a keyword string into a list of terms. When fewer+// than 3 comma terms exist, falls back to space-split bigrams (mirrors Python).+func splitKeywords(keywords string) []string {+ if strings.TrimSpace(keywords) == "" {+ return nil+ }+ kwds := make([]string, 0, 8)+ for _, k := range strings.Split(keywords, ",") {+ if k = strings.TrimSpace(k); k != "" {+ kwds = append(kwds, strings.ToLower(k))+ }+ }+ if len(kwds) < 3 {+ words := make([]string, 0, 8)+ for _, w := range strings.Split(keywords, " ") {+ if w = strings.TrimSpace(w); w != "" {+ words = append(words, strings.ToLower(w))+ }+ }+ bigrams := make([]string, 0, len(words))+ for i := 0; i+1 < len(words); i++ {+ bigrams = append(bigrams, words[i]+" "+words[i+1])+ }+ if len(bigrams) > 0 {+ return bigrams+ }+ }+ return kwds+}++var sentEnd = regexp.MustCompile(`[。!?;!?;]+|\.`)++// splitSentences splits text into sentences at sentence terminators, keeping a+// digit-guarded period intact ("3.14" / "v1.2" are not split). Implemented with+// a simple splitter because RE2 (Go) does not support lookbehind/lookahead.+func splitSentences(content string) []string {+ raw := sentEnd.Split(content, -1)+ sents := make([]string, 0, len(raw))+ for _, s := range raw {+ if strings.TrimSpace(s) == "" {+ continue+ }+ // Re-join a trailing digit-period-digit that the splitter cut apart:+ // if s ends with a digit and content had ".<digit>" following, reattach.+ sents = append(sents, s)+ }+ return rejoinDigitPeriods(sents)+}++// rejoinDigitPeriods merges "…1" + "2…" back into "…1.2…" when a decimal point+// separated two digit groups.+func rejoinDigitPeriods(sents []string) []string {+ out := make([]string, 0, len(sents))+ for i := 0; i < len(sents); i++ {+ cur := sents[i]+ // If current ends with a digit and next begins with a digit, the split+ // point was a decimal point — merge them.+ for i+1 < len(sents) && hasTrailingDigit(cur) && hasLeadingDigit(sents[i+1]) {+ cur = strings.TrimRight(cur, " \t") + "." + sents[i+1]+ i+++ }+ out = append(out, cur)+ }+ return out+}++func hasTrailingDigit(s string) bool {+ s = strings.TrimRight(s, " \t")+ return len(s) > 0 && s[len(s)-1] >= '0' && s[len(s)-1] <= '9'+}++func hasLeadingDigit(s string) bool {+ s = strings.TrimLeft(s, " \t")+ return len(s) > 0 && s[0] >= '0' && s[0] <= '9'+}++// narrowContent returns the keyword-bearing sentences (+/-1 neighbour) with the+// keyword highlighted, or (_, false) if no keyword occurs.+func narrowContent(content string, kwds []string) (string, bool) {+ if strings.TrimSpace(content) == "" {+ return "", false+ }+ sents := splitSentences(content)+ if len(sents) == 0 {+ return "", false+ }+ keep := map[int]bool{}+ matched := false+ for i, s := range sents {+ low := strings.ToLower(s)+ for _, kw := range kwds {+ if kw != "" && strings.Contains(low, kw) {+ matched = true+ if i > 0 {+ keep[i-1] = true+ }+ keep[i] = true+ if i+1 < len(sents) {+ keep[i+1] = true+ }+ break+ }+ }+ }+ if !matched {+ return "", false+ }+ var b strings.Builder+ for i := 0; i < len(sents); i++ {+ if keep[i] {+ b.WriteString(sents[i])+ }+ }+ return "..." + highlightKeywords(b.String(), kwds) + "...", true+}++// highlightKeywords wraps keyword occurrences in <em>.+func highlightKeywords(text string, kwds []string) string {+ if len(kwds) == 0 {+ return text+ }+ // Sort by length desc so longer terms match first.+ terms := make([]string, len(kwds))+ copy(terms, kwds)+ for i := 1; i < len(terms); i++ {+ for j := i; j > 0 && len(terms[j]) > len(terms[j-1]); j-- {+ terms[j], terms[j-1] = terms[j-1], terms[j]+ }+ }+ pattern := "("+ for i, t := range terms {+ if t == "" {+ continue+ }+ if i > 0 {+ pattern += "|"+ }+ pattern += regexp.QuoteMeta(t)+ }+ pattern += ")"+ re := regexp.MustCompile(`(?i)` + pattern)+ return re.ReplaceAllString(text, "<em>${1}</em>")+}++func md5Hex(s string) string {+ h := uint32(2166136261)+ for i := 0; i < len(s); i++ {+ h ^= uint32(s[i])+ h *= 16777619+ }+ return fmt.Sprintf("%08x", h)+}++func jsonChunksEmpty() string {+ return `{"chunks":[]}`+}++func marshalSearchResult(chunks []RetrievalChunk) string {+ type outChunk struct {+ ID string `json:"id"`+ Content string `json:"content"`+ DocumentID string `json:"doc_id"`+ DocName string `json:"docnm_kwd"`+ Score float64 `json:"similarity"`+ }+ out := make([]outChunk, 0, len(chunks))+ for _, c := range chunks {+ out = append(out, outChunk{+ ID: c.ID, Content: c.Content, DocumentID: c.DocumentID,+ DocName: c.DocumentName, Score: c.Score,+ })+ }+ b, err := json.Marshal(map[string]interface{}{"chunks": out})+ if err != nil {+ return jsonChunksEmpty()+ }+ return string(b)+}diff --git a/internal/agent/tool/agentic_search_test.go b/internal/agent/tool/agentic_search_test.gonew file mode 100644index 0000000000..c0c49baf91--- /dev/null+++ b/internal/agent/tool/agentic_search_test.go@@ -0,0 +1,78 @@+package tool++import (+ "strings"+ "testing"+)++// TestSplitKeywords_BigramFallback asserts comma terms are used when >=3, and+// space-split bigrams otherwise.+func TestSplitKeywords_BigramFallback(t *testing.T) {+ got := splitKeywords("alpha, beta, gamma")+ if len(got) != 3 {+ t.Fatalf("comma kwds = %d, want 3", len(got))+ }+ got = splitKeywords("a b c d")+ // 4 words -> 3 bigrams+ if len(got) != 3 {+ t.Fatalf("bigram kwds = %d, want 3", len(got))+ }+ if got[0] != "a b" {+ t.Errorf("bigram[0] = %q, want \"a b\"", got[0])+ }+}++// TestNarrowContent_KeepsKeywordSentence asserts narrowing keeps the+// keyword-bearing sentence and its neighbours, and highlights the keyword.+func TestNarrowContent_KeepsKeywordSentence(t *testing.T) {+ content := "The introduction is boring. The key insight about rocket engines is here. The conclusion is short."+ nc, ok := narrowContent(content, []string{"rocket"})+ if !ok {+ t.Fatal("expected keyword match")+ }+ if !strings.Contains(nc, "key insight") {+ t.Errorf("narrowed content missing keyword sentence: %q", nc)+ }+ if !strings.Contains(nc, "<em>rocket</em>") {+ t.Errorf("narrowed content missing highlight: %q", nc)+ }+}++// TestNarrowContent_NoKeyword asserts narrowing returns false when no keyword+// occurs.+func TestNarrowContent_NoKeyword(t *testing.T) {+ if _, ok := narrowContent("no match here at all", []string{"zzz"}); ok {+ t.Fatal("expected no match")+ }+}++// TestSplitSentences_DigitPeriod asserts "3.14" is not split into two sentences.+func TestSplitSentences_DigitPeriod(t *testing.T) {+ sents := splitSentences("pi is 3.14 and e is 2.71. end.")+ found := false+ for _, s := range sents {+ if strings.Contains(s, "3.14") {+ found = true+ }+ }+ if !found {+ t.Fatalf("digit-period sentence lost: %v", sents)+ }+}++// TestNarrowByKeywords_DropsKeywordless asserts chunks without keywords are+// dropped and deduped by narrowed content hash.+func TestNarrowByKeywords_DropsKeywordless(t *testing.T) {+ chunks := []RetrievalChunk{+ {ID: "c1", Content: "alpha talks about beta in detail here."},+ {ID: "c2", Content: "unrelated content entirely."},+ {ID: "c3", Content: "alpha talks about beta in detail here."}, // dup of c1+ }+ out := narrowByKeywords(chunks, "beta")+ if len(out) != 1 {+ t.Fatalf("narrowed chunks = %d, want 1 (drops keywordless + dedups)", len(out))+ }+ if out[0].ID != "c1" {+ t.Errorf("kept chunk = %q, want c1", out[0].ID)+ }+}diff --git a/internal/agent/tool/canvas_ctx.go b/internal/agent/tool/canvas_ctx.gonew file mode 100644index 0000000000..6fb4d78687--- /dev/null+++ b/internal/agent/tool/canvas_ctx.go@@ -0,0 +1,59 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package tool++import (+ "context"++ "ragflow/internal/agent/runtime"+)++// canvasTenantID derives the tenant id from canvas state, falling back to+// user_id. Shared by agentic search and dataset-navigation tools.+func canvasTenantID(ctx context.Context) string {+ state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx)+ if err != nil || state == nil {+ return ""+ }+ if tenantID, _ := state.Sys["tenant_id"].(string); tenantID != "" {+ return tenantID+ }+ userID, _ := state.Sys["user_id"].(string)+ return userID+}++// canvasDatasetIDs returns the explicit dataset ids (all of them, preserving+// multi-KB sessions), else the canvas sys dataset_id as a single-element list.+func canvasDatasetIDs(ctx context.Context, explicit []string) []string {+ if len(explicit) > 0 {+ out := make([]string, 0, len(explicit))+ for _, id := range explicit {+ if id != "" {+ out = append(out, id)+ }+ }+ return out+ }+ state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx)+ if err != nil || state == nil {+ return nil+ }+ if id, _ := state.Sys["dataset_id"].(string); id != "" {+ return []string{id}+ }+ return nil+}diff --git a/internal/agent/tool/dataset_navigation.go b/internal/agent/tool/dataset_navigation.gonew file mode 100644index 0000000000..0f33b6749e--- /dev/null+++ b/internal/agent/tool/dataset_navigation.go@@ -0,0 +1,220 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package tool++import (+ "context"+ "encoding/json"+ "fmt"+ "strings"++ einotool "github.com/cloudwego/eino/components/tool"+ "github.com/cloudwego/eino/schema"++ "ragflow/internal/service/nav"+)++// datasetNavigationToolName mirrors Python's dataset_navigation_by_tree router+// tool. It navigates the dataset nav tree and returns the doc_ids to read.+const datasetNavigationToolName = "dataset_navigation_by_tree"++const datasetNavigationToolDescription = "Navigate a dataset's navigation tree by topic and return the document ids that are likely relevant."++// datasetNavigationArgs is the JSON schema the model sends into InvokableRun.+type datasetNavigationArgs struct {+ Topic string `json:"topic"`+ Keywords string `json:"keywords,omitempty"`+ DatasetIDs []string `json:"dataset_ids,omitempty"`+ DocScope string `json:"doc_scope,omitempty"`+ MaxDocs int `json:"max_docs,omitempty"`+}++// datasetNavigationResult is the JSON shape returned to the model.+type datasetNavigationResult struct {+ Docs []string `json:"docs,omitempty"`+ Error string `json:"_ERROR,omitempty"`+ NotFound bool `json:"not_found,omitempty"`+}++// datasetNavigationDefaultMaxDocs caps the number of doc_ids returned.+const datasetNavigationDefaultMaxDocs = 8++// DatasetNavigationByTree is the dataset-navigation router tool. Minimal closed+// loop: one-level drill-down from the root clusters and deduplicated doc ids+// (max MaxDocs). LLM-guided multi-level selection is deferred.+type DatasetNavigationByTree struct {+ defaults datasetNavigationArgs+}++// NewDatasetNavigationByTree returns a DatasetNavigationByTree implementing+// eino's tool.InvokableTool interface.+func NewDatasetNavigationByTree() *DatasetNavigationByTree {+ return NewDatasetNavigationByTreeWithDefaults(datasetNavigationArgs{})+}++// NewDatasetNavigationByTreeWithDefaults returns a DatasetNavigationByTree with+// node-level defaults.+func NewDatasetNavigationByTreeWithDefaults(defaults datasetNavigationArgs) *DatasetNavigationByTree {+ if defaults.MaxDocs <= 0 {+ defaults.MaxDocs = datasetNavigationDefaultMaxDocs+ }+ return &DatasetNavigationByTree{defaults: defaults}+}++// Info returns the tool's metadata for the chat model.+func (d *DatasetNavigationByTree) Info(_ context.Context) (*schema.ToolInfo, error) {+ return &schema.ToolInfo{+ Name: datasetNavigationToolName,+ Desc: datasetNavigationToolDescription,+ ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{+ "topic": {+ Type: schema.String,+ Desc: "The topic to navigate to. Use the core subject from the original request.",+ Required: true,+ },+ "keywords": {+ Type: schema.String,+ Desc: "Optional additional keywords to disambiguate the topic.",+ },+ }),+ }, nil+}++// InvokableRun executes the tool. It navigates the nav tree via the registered+// NavService (internal/service datasetnav) and returns a deduplicated doc_id+// list (max MaxDocs).+func (d *DatasetNavigationByTree) InvokableRun(ctx context.Context, argumentsInJSON string, _ ...einotool.Option) (string, error) {+ var args datasetNavigationArgs+ if argumentsInJSON != "" {+ if err := json.Unmarshal([]byte(argumentsInJSON), &args); err != nil {+ return "", fmt.Errorf("dataset_navigation: parse arguments: %w", err)+ }+ }+ args = d.mergeDefaults(args)+ if args.Topic == "" {+ return "", fmt.Errorf("dataset_navigation: topic is required")+ }+ // Per-request max_docs overrides the node default; default to a sane cap.+ maxDocs := args.MaxDocs+ if maxDocs <= 0 {+ maxDocs = datasetNavigationDefaultMaxDocs+ }++ ns := nav.GetNavService()+ if ns == nil {+ return datasetNavigationJSON(datasetNavigationResult{+ Error: "dataset navigation service not initialized (SetNavService must be called at bootstrap)",+ }), nil+ }++ tenantID := canvasTenantID(ctx)+ datasetIDs := canvasDatasetIDs(ctx, args.DatasetIDs)+ if tenantID == "" || len(datasetIDs) == 0 {+ return datasetNavigationJSON(datasetNavigationResult{+ NotFound: true,+ Error: "dataset navigation requires a tenant and dataset context",+ }), nil+ }++ // Route RELEVANT docs by querying the nav tree with the topic (semantic KNN).+ // The topic is the routing signal — we must not return arbitrary doc ids.+ query := strings.TrimSpace(args.Topic + " " + args.Keywords)+ seen := map[string]struct{}{}+ var docs []string+ collect := func(id string) {+ if id == "" {+ return+ }+ if _, ok := seen[id]; ok {+ return+ }+ if len(docs) >= maxDocs {+ return+ }+ seen[id] = struct{}{}+ docs = append(docs, id)+ }++ // Primary: semantic search over each dataset's nav tree.+ for _, datasetID := range datasetIDs {+ hits, err := ns.Search(ctx, tenantID, datasetID, query, nil, maxDocs)+ if err != nil {+ continue+ }+ for _, h := range hits {+ collect(h.DocID)+ for _, id := range h.DocIDs {+ collect(id)+ }+ }+ if len(docs) >= maxDocs {+ break+ }+ }++ // Fallback: if semantic routing found nothing (e.g. no embedder), walk the+ // root clusters so the tool still returns a useful (if coarse) doc set.+ if len(docs) == 0 {+ for _, datasetID := range datasetIDs {+ clusters, _, err := ns.ListClusters(ctx, tenantID, datasetID, 0, 100)+ if err != nil {+ continue+ }+ for _, c := range clusters {+ children, _, err := ns.ListChildren(ctx, tenantID, datasetID, c.Name, 0, 100)+ if err != nil {+ continue+ }+ for _, ch := range children {+ collect(ch.DocID)+ if len(docs) >= maxDocs {+ break+ }+ }+ if len(docs) >= maxDocs {+ break+ }+ }+ if len(docs) >= maxDocs {+ break+ }+ }+ }++ if len(docs) == 0 {+ return datasetNavigationJSON(datasetNavigationResult{NotFound: true}), nil+ }+ return datasetNavigationJSON(datasetNavigationResult{Docs: docs}), nil+}++func (d *DatasetNavigationByTree) mergeDefaults(args datasetNavigationArgs) datasetNavigationArgs {+ if len(args.DatasetIDs) == 0 && len(d.defaults.DatasetIDs) != 0 {+ args.DatasetIDs = append([]string(nil), d.defaults.DatasetIDs...)+ }+ if args.MaxDocs <= 0 {+ args.MaxDocs = d.defaults.MaxDocs+ }+ return args+}++func datasetNavigationJSON(r datasetNavigationResult) string {+ b, err := json.Marshal(r)+ if err != nil {+ return fmt.Sprintf(`{"_ERROR":"dataset_navigation: marshal result: %s"}`, err)+ }+ return string(b)+}diff --git a/internal/agent/tool/dataset_navigation_test.go b/internal/agent/tool/dataset_navigation_test.gonew file mode 100644index 0000000000..3f7b8dbff5--- /dev/null+++ b/internal/agent/tool/dataset_navigation_test.go@@ -0,0 +1,134 @@+package tool++import (+ "context"+ "sync"+ "testing"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/service/nav"+)++// navRoutingFake is a nav.NavService that records Search calls (topic) and+// returns a controlled doc list, so a test can assert the router actually+// queries by topic rather than walking arbitrary clusters.+type navRoutingFake struct {+ mu sync.Mutex+ searched []string // topics passed to Search+ hits []nav.NavHit+ clusters []nav.NavNode+ children map[string][]nav.NavNode+}++func (f *navRoutingFake) UpsertDoc(context.Context, nav.UpsertDocInput) error { return nil }+func (f *navRoutingFake) RemoveDoc(context.Context, string, string, string) error {+ return nil+}+func (f *navRoutingFake) Search(_ context.Context, _, _ string, query string, _ []float32, _ int) ([]nav.NavHit, error) {+ f.mu.Lock()+ f.searched = append(f.searched, query)+ f.mu.Unlock()+ return f.hits, nil+}+func (f *navRoutingFake) ListClusters(context.Context, string, string, int, int) ([]nav.NavNode, int64, error) {+ return f.clusters, int64(len(f.clusters)), nil+}+func (f *navRoutingFake) ListChildren(_ context.Context, _, _, name string, _, _ int) ([]nav.NavNode, int64, error) {+ return f.children[name], int64(len(f.children[name])), nil+}+func (f *navRoutingFake) searchedTopics() []string {+ f.mu.Lock()+ defer f.mu.Unlock()+ return append([]string(nil), f.searched...)+}++// TestDatasetNavigation_UsesTopicRouting asserts the router queries the nav tree+// with the topic (semantic search), rather than blindly walking clusters and+// returning arbitrary doc ids.+func TestDatasetNavigation_UsesTopicRouting(t *testing.T) {+ fake := &navRoutingFake{+ hits: []nav.NavHit{+ {Type: "nav_doc", DocID: "d1", Name: "rocket"},+ {Type: "nav_doc", DocID: "d2", Name: "engine"},+ },+ }+ prev := nav.GetNavService()+ nav.SetNavService(fake)+ defer func() { nav.SetNavService(prev) }()++ state := runtime.NewCanvasState("run-1", "task-1")+ state.Sys["tenant_id"] = "tenant-1"+ ctx := runtime.WithState(context.Background(), state)++ tool := NewDatasetNavigationByTree()+ out, err := tool.InvokableRun(ctx, `{"topic":"rocket propulsion","keywords":"engine","dataset_ids":["kb1"]}`)+ if err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ // The topic (plus keywords) must have been used as the Search query.+ topics := fake.searchedTopics()+ if len(topics) == 0 {+ t.Fatal("Search was never called; router must route by topic")+ }+ if topics[0] != "rocket propulsion engine" {+ t.Errorf("search query = %q, want topic+keywords", topics[0])+ }+ // The returned docs come from the relevant hits, not arbitrary walk.+ if !containsStr(out, "d1") || !containsStr(out, "d2") {+ t.Errorf("routed docs missing hits: %s", out)+ }+}++// TestCanvasDatasetIDs_MultiKB asserts all explicit dataset ids are preserved+// (a multi-KB session must not collapse to the first KB).+func TestCanvasDatasetIDs_MultiKB(t *testing.T) {+ ids := canvasDatasetIDs(context.Background(), []string{"kb1", "kb2", "kb3"})+ if len(ids) != 3 || ids[0] != "kb1" || ids[1] != "kb2" || ids[2] != "kb3" {+ t.Errorf("canvasDatasetIDs = %v, want all three KBs", ids)+ }+}++// TestDatasetNavigation_MultiKB asserts the router searches EVERY bound dataset+// (not just the first), so docs in other KBs stay reachable.+func TestDatasetNavigation_MultiKB(t *testing.T) {+ fake := &navRoutingFake{hits: []nav.NavHit{{Type: "nav_doc", DocID: "d1", Name: "topic"}}}+ prev := nav.GetNavService()+ nav.SetNavService(fake)+ defer func() { nav.SetNavService(prev) }()++ state := runtime.NewCanvasState("run-1", "task-1")+ state.Sys["tenant_id"] = "tenant-1"+ ctx := runtime.WithState(context.Background(), state)++ tool := NewDatasetNavigationByTree()+ _, err := tool.InvokableRun(ctx, `{"topic":"X","dataset_ids":["kb1","kb2","kb3"]}`)+ if err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ // Search must have been called once per dataset (3 calls), not collapsed to+ // the first KB.+ if got := len(fake.searchedTopics()); got != 3 {+ t.Errorf("Search called %d times, want 3 (once per dataset)", got)+ }+}++// TestCanvasDatasetIDs_DedupEmpty asserts empty ids are dropped.+func TestCanvasDatasetIDs_DedupEmpty(t *testing.T) {+ ids := canvasDatasetIDs(context.Background(), []string{"kb1", "", "kb2"})+ if len(ids) != 2 || ids[0] != "kb1" || ids[1] != "kb2" {+ t.Errorf("canvasDatasetIDs = %v, want [kb1 kb2]", ids)+ }+}++func containsStr(s, sub string) bool {+ return len(s) > 0 && len(sub) > 0 && (s == sub || containsSub(s, sub))+}++func containsSub(s, sub string) bool {+ for i := 0; i+len(sub) <= len(s); i++ {+ if s[i:i+len(sub)] == sub {+ return true+ }+ }+ return false+}diff --git a/internal/agent/tool/registry.go b/internal/agent/tool/registry.goindex ca43ca7830..674e831a6e 100644--- a/internal/agent/tool/registry.go+++ b/internal/agent/tool/registry.go@@ -31,32 +31,36 @@ import (type Factory func(params map[string]any) (einotool.BaseTool, error)var registry = map[string]Factory{- "akshare": buildAkShareTool,- "arxiv": buildArxivTool,- "bgpt": buildBGPTTool,- "code_exec": noConfig("code_exec", func() einotool.BaseTool { return NewCodeExecTool() }),- "crawler": noConfig("crawler", func() einotool.BaseTool { return NewCrawlerTool() }),- "deepl": noConfig("deepl", func() einotool.BaseTool { return NewDeepLTool() }),- "duckduckgo": buildDuckDuckGoTool,- "email": buildEmailTool,- "execute_sql": buildExeSQLTool,- "exesql": buildExeSQLTool,- "github": buildGitHubTool,- "google": buildGoogleTool,- "google_scholar": buildGoogleScholarTool,- "google_scholar_search": buildGoogleScholarTool,- "jin10": noConfig("jin10", func() einotool.BaseTool { return NewJin10Tool() }),- "keenable": buildKeenableTool,- "pubmed": buildPubMedTool,- "qweather": noConfig("qweather", func() einotool.BaseTool { return NewQWeatherTool() }),- "querit": buildQueritTool,- "querit_search": buildQueritTool,- "queritsearch": buildQueritTool,- "retrieval": buildRetrievalTool,- "search_my_dataset": buildRetrievalTool,- "search_my_dateset": buildRetrievalTool,- "searxng": buildSearXNGTool,- "tavily": buildTavilyTool,+ "akshare": buildAkShareTool,+ "arxiv": buildArxivTool,+ "bgpt": buildBGPTTool,+ "code_exec": noConfig("code_exec", func() einotool.BaseTool { return NewCodeExecTool() }),+ "crawler": noConfig("crawler", func() einotool.BaseTool { return NewCrawlerTool() }),+ "dataset_navigation_by_tree": noConfig("dataset_navigation_by_tree", func() einotool.BaseTool { return NewDatasetNavigationByTree() }),+ "hybrid_search": noConfig("hybrid_search", func() einotool.BaseTool { return NewAgenticSearchTool(toolHybridSearch) }),+ "vector_search": noConfig("vector_search", func() einotool.BaseTool { return NewAgenticSearchTool(toolVectorSearch) }),+ "bm25_search": noConfig("bm25_search", func() einotool.BaseTool { return NewAgenticSearchTool(toolBM25Search) }),+ "deepl": noConfig("deepl", func() einotool.BaseTool { return NewDeepLTool() }),+ "duckduckgo": buildDuckDuckGoTool,+ "email": buildEmailTool,+ "execute_sql": buildExeSQLTool,+ "exesql": buildExeSQLTool,+ "github": buildGitHubTool,+ "google": buildGoogleTool,+ "google_scholar": buildGoogleScholarTool,+ "google_scholar_search": buildGoogleScholarTool,+ "jin10": noConfig("jin10", func() einotool.BaseTool { return NewJin10Tool() }),+ "keenable": buildKeenableTool,+ "pubmed": buildPubMedTool,+ "qweather": noConfig("qweather", func() einotool.BaseTool { return NewQWeatherTool() }),+ "querit": buildQueritTool,+ "querit_search": buildQueritTool,+ "queritsearch": buildQueritTool,+ "retrieval": buildRetrievalTool,+ "search_my_dataset": buildRetrievalTool,+ "search_my_dateset": buildRetrievalTool,+ "searxng": buildSearXNGTool,+ "tavily": buildTavilyTool,// Agent DSL tool lists carry the Python Canvas component_name verbatim.// BuildByName lower-cases names, so register those component names too."tavilysearch": buildTavilyTool,diff --git a/internal/agent/tool/retrieval_nlp.go b/internal/agent/tool/retrieval_nlp.goindex 88f9e14d72..2a15193b3d 100644--- a/internal/agent/tool/retrieval_nlp.go+++ b/internal/agent/tool/retrieval_nlp.go@@ -162,6 +162,7 @@ func nlpRequestFromRetrieval(req RetrievalRequest, tenantIDs []string, topN int)Question: req.Query,TenantIDs: append([]string(nil), tenantIDs...),KbIDs: append([]string(nil), req.DatasetIDs...),+ DocIDs: append([]string(nil), compactStrings(req.DocScope)...),Page: 1,PageSize: topN,Aggs: boolPtr(false),diff --git a/internal/agent/tool/retrieval_service.go b/internal/agent/tool/retrieval_service.goindex ecc0615987..9b44733acc 100644--- a/internal/agent/tool/retrieval_service.go+++ b/internal/agent/tool/retrieval_service.go@@ -55,6 +55,9 @@ type RetrievalRequest struct {KeywordsSimilarityWeight *float64UseKG boolSimilarityThreshold float64+ // DocScope restricts retrieval to a set of document ids (the doc_id list+ // routed by the dataset_navigation_by_tree tool). Empty = no doc filter.+ DocScope []string// TenantID is the calling tenant (== user_id in RAGFlow's data model).// Optional for the nlp adapter; the KG adapter uses it to resolve the// tenant's default chat + embedding models. Reads fromdiff --git a/internal/dao/compilation_template_seed.go b/internal/dao/compilation_template_seed.goindex 653a1f8e31..38c8d690d6 100644--- a/internal/dao/compilation_template_seed.go+++ b/internal/dao/compilation_template_seed.go@@ -43,7 +43,6 @@ var builtinCompilationTemplateKinds = []struct {{Kind: "mind_map", Name: "Mind map - Radial concept hierarchy"},{Kind: "wiki", Name: "Wiki — Graph-based wiki"},{Kind: "knowledge_graph", Name: "Knowledge graph"},- {Kind: "datasetnav", Name: "Dataset nav — top-down drill-down"},{Kind: "page_index", Name: "Page index"},{Kind: "session_essence", Name: "Session essence"},{Kind: "session_graph", Name: "Session graph"},diff --git a/internal/dao/compilation_template_seed_test.go b/internal/dao/compilation_template_seed_test.goindex d12fb76cd1..dca74cd70c 100644--- a/internal/dao/compilation_template_seed_test.go+++ b/internal/dao/compilation_template_seed_test.go@@ -25,6 +25,18 @@ import ("gorm.io/gorm")+// TestBuiltinCompilationTemplateKinds_NoDatasetnav locks the removal of the+// datasetnav template from the built-in catalogue: dataset navigation is not an+// independent compile kind (see component_test.TestKnowledgeCompiler_Datasetnav_NoVariant),+// so it must not appear in the seeded template kinds.+func TestBuiltinCompilationTemplateKinds_NoDatasetnav(t *testing.T) {+ for _, k := range builtinCompilationTemplateKinds {+ if k.Kind == "datasetnav" || k.Kind == "dataset_nav" {+ t.Fatalf("builtin compilation template kind %q must not exist (datasetnav is a by-product, not a compile kind)", k.Kind)+ }+ }+}+func TestSeedBuiltinCompilationTemplatesForTenant(t *testing.T) {db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})if err != nil {diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.goindex 97375efcb0..7c0635566e 100644--- a/internal/ingestion/component/knowledge_compiler/common/types.go+++ b/internal/ingestion/component/knowledge_compiler/common/types.go@@ -18,11 +18,10 @@ import (type Variant stringconst (- VariantStructure Variant = "structure"- VariantWiki Variant = "wiki"- VariantTree Variant = "tree"- VariantMindmap Variant = "mindmap"- VariantDatasetnav Variant = "datasetnav"+ VariantStructure Variant = "structure"+ VariantWiki Variant = "wiki"+ VariantTree Variant = "tree"+ VariantMindmap Variant = "mindmap")// Sentinel errors.@@ -189,10 +188,13 @@ func ParseParam(m map[string]any) (Param, error) {// knowledge_graph ->//// The canonical variant names are also accepted as identity (so a template kind-// may equal the variant, e.g. "datasetnav" which has no Python kind, and the-// internal unit tests can drive each variant through a template id). Unknown-// kinds return ErrUnknownVariant; the caller turns that into a hard failure-// rather than silently emitting uncompiled rows.+// may equal the variant). Unknown kinds return ErrUnknownVariant; the caller+// turns that into a hard failure rather than silently emitting uncompiled rows.+//+// Note: "datasetnav"/"dataset_nav" intentionally has NO mapping here — dataset+// navigation is not an independent compile kind in Python; it is a by-product+// written after tree/page_index compilation via internal/service datasetnav+// (see tasks/agentic_search_port_plan.md).func KindToVariant(kind string) (Variant, error) {switch strings.ToLower(strings.TrimSpace(kind)) {case "tree":@@ -201,8 +203,6 @@ func KindToVariant(kind string) (Variant, error) {return VariantMindmap, nilcase "wiki":return VariantWiki, nil- case "datasetnav", "dataset_nav":- return VariantDatasetnav, nilcase "page_index", "session_essence", "session_graph", "timeline","knowledge_graph", "structure", "knowledgegraph", "graph":return VariantStructure, nildiff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex 17724b98ba..bb870f4b93 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -1,22 +1,24 @@// Package knowledge_compiler implements the KnowledgeCompiler ingestion// component: a single runtime.Component that dispatches to one of the-// knowledge-compile variants (structure / wiki / tree / mindmap / datasetnav)-// based on the `variant` param. See PORT_PLAN.md for the full design.+// knowledge-compile variants (structure / wiki / tree / mindmap) based on the+// `variant` param. See PORT_PLAN.md for the full design.package knowledge_compilerimport ("context"+ "encoding/json""fmt"+ "log""strings""ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/knowledge_compiler/common"- "ragflow/internal/ingestion/component/knowledge_compiler/datasetnav""ragflow/internal/ingestion/component/knowledge_compiler/mindmap""ragflow/internal/ingestion/component/knowledge_compiler/structure""ragflow/internal/ingestion/component/knowledge_compiler/tree""ragflow/internal/ingestion/component/knowledge_compiler/wiki""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/service/nav""ragflow/internal/tokenizer""gorm.io/gorm"@@ -110,6 +112,9 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in// template's id and kind. All products are buffered (no streaming sink), so// the post-run loop below covers every row (M1).var out common.Outputs+ // navByProducts accumulates every tree/structure by-product job so each is+ // written after the spec loop (not just the last one).+ var navByProducts []navByProductfor _, spec := range specs {variant, err := common.KindToVariant(spec.Kind)if err != nil {@@ -150,8 +155,6 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, ino, err = tree.Run(ctx, deps, specParam, specIn)case common.VariantMindmap:o, err = mindmap.Run(ctx, deps, specParam, specIn)- case common.VariantDatasetnav:- o, err = datasetnav.Run(ctx, deps, specParam, specIn)default:return nil, fmt.Errorf("%w: %q", common.ErrUnknownVariant, variant)}@@ -159,6 +162,13 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inreturn nil, err}+ // Accumulate tree/structure by-product jobs so each is written after the+ // spec loop (a multi-spec batch must not drop any spec's products). nav is+ // a derived artifact; its failures are logged and never abort the pipeline.+ if variant == common.VariantTree || variant == common.VariantStructure {+ navByProducts = append(navByProducts, navByProduct{deps: deps, variant: variant, products: o.Products})+ }+for i := range o.Products {if o.Products[i].Meta == nil {o.Products[i].Meta = map[string]any{}@@ -171,6 +181,17 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inout.Products = append(out.Products, o.Products...)}+ // Write each dataset-nav by-product after all specs ran. tree by-product+ // (Python compiler.py:475) and structure/page_index by-product (Python+ // runner.py:189). Failures are logged and never abort the pipeline.+ for _, job := range navByProducts {+ if job.variant == common.VariantTree {+ upsertTreeNav(ctx, job.deps, in.DocID, job.products)+ } else {+ upsertStructureNav(ctx, job.deps, in.DocID, job.products)+ }+ }+// Convert the compiled products into chunk-aligned docs (matching// conf/infinity_mapping.json) and merge them into the upstream input// chunks. The component stays DB-independent and no longer routes through a@@ -182,6 +203,142 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inreturn mergeChunks(inputs, compiled), nil}+// navByProduct is one deferred dataset-nav by-product job: the compile variant+// that produced it and the products to summarize from.+type navByProduct struct {+ deps common.Deps+ variant common.Variant+ products []common.Product+}++// upsertTreeNav writes the dataset-nav by-product after tree compilation. It+// finds the tree root product (Meta kind=root) whose Content is the document+// summary, and places it into the ES-backed nav tree. Any failure (missing nav+// service, missing root, embedding/upsert error) is logged and skipped — the+// nav artifact must never block the compile pipeline.+func upsertTreeNav(ctx context.Context, deps common.Deps, docID string, products []common.Product) {+ ns := nav.GetNavService()+ if ns == nil {+ log.Printf("knowledge_compiler: datasetnav by-product skipped (NavService not initialized)")+ return+ }+ var summary string+ var vec []float32+ for i := range products {+ if kind, _ := products[i].Meta["kind"].(string); kind == "root" {+ summary = products[i].Content+ vec = products[i].Vector+ break+ }+ }+ if strings.TrimSpace(summary) == "" {+ log.Printf("knowledge_compiler: datasetnav by-product skipped (tree has no root summary)")+ return+ }+ if len(vec) == 0 {+ if deps.Embed == nil {+ log.Printf("knowledge_compiler: datasetnav by-product skipped (no embedder, no precomputed vector)")+ return+ }+ embeddings, err := deps.Embed.Encode(ctx, []string{summary})+ if err != nil {+ log.Printf("knowledge_compiler: datasetnav by-product skipped (embedding failed): %v", err)+ return+ }+ if len(embeddings) == 0 {+ log.Printf("knowledge_compiler: datasetnav by-product skipped (embedding produced no vector)")+ return+ }+ vec = embeddings[0]+ }+ if err := ns.UpsertDoc(ctx, nav.UpsertDocInput{+ TenantID: deps.TenantID,+ KbID: deps.DatasetID,+ DocID: docID,+ Summary: summary,+ Embedd: vec,+ }); err != nil {+ // Log and continue: nav is a derived artifact, never abort compile.+ log.Printf("knowledge_compiler: datasetnav by-product upsert failed (continuing): %v", err)+ }+}++// upsertStructureNav writes the dataset-nav by-product after structure/page_index+// compilation (mirroring Python runner.py:189 _upsert_dataset_nav_from_page_index).+// It finds the graph product (Meta kind=graph), summarizes its entity+// descriptions into a document-level summary, and places it into the nav tree.+// Failures are logged and never abort the compile pipeline.+func upsertStructureNav(ctx context.Context, deps common.Deps, docID string, products []common.Product) {+ ns := nav.GetNavService()+ if ns == nil {+ log.Printf("knowledge_compiler: datasetnav by-product skipped (NavService not initialized)")+ return+ }+ var graph *common.Product+ for i := range products {+ if kind, _ := products[i].Meta["kind"].(string); kind == "graph" {+ graph = &products[i]+ break+ }+ }+ if graph == nil {+ return+ }+ summary := pageIndexSummary(graph.Content)+ if strings.TrimSpace(summary) == "" {+ log.Printf("knowledge_compiler: datasetnav by-product skipped (structure graph has no entity descriptions)")+ return+ }+ // Embed the SUMMARY text, not the graph JSON — the nav doc's vector must+ // represent the summary semantics for the KNN router to match correctly.+ if deps.Embed == nil {+ log.Printf("knowledge_compiler: datasetnav by-product skipped (no embedder)")+ return+ }+ embeddings, err := deps.Embed.Encode(ctx, []string{summary})+ if err != nil || len(embeddings) == 0 {+ log.Printf("knowledge_compiler: datasetnav by-product skipped (structure embedding failed): %v", err)+ return+ }+ vec := embeddings[0]+ if err := ns.UpsertDoc(ctx, nav.UpsertDocInput{+ TenantID: deps.TenantID,+ KbID: deps.DatasetID,+ DocID: docID,+ Summary: summary,+ Embedd: vec,+ }); err != nil {+ log.Printf("knowledge_compiler: datasetnav by-product upsert failed (continuing): %v", err)+ }+}++// pageIndexSummary concatenates entity descriptions from a structure graph JSON+// ({"entities": [{"name","description"}, ...]}), producing a document-level+// summary for dataset navigation.+func pageIndexSummary(graphJSON string) string {+ var graph struct {+ Entities []struct {+ Name string `json:"name"`+ Description string `json:"description"`+ } `json:"entities"`+ }+ if err := json.Unmarshal([]byte(graphJSON), &graph); err != nil {+ return ""+ }+ var b strings.Builder+ for _, e := range graph.Entities {+ desc := strings.Join(strings.Fields(e.Description), " ")+ if desc == "" {+ continue+ }+ if e.Name != "" {+ b.WriteString(e.Name + ": ")+ }+ b.WriteString(desc + "\n")+ }+ return b.String()+}+// resolveTemplateSpecs resolves the configured compilation template spec(s) to// their TemplateInfo rows. Priority: compilation_template_id >// compilation_template_group_id. The group path resolves the group to its@@ -260,11 +417,10 @@ func kindOrVariant(p common.Product) string {// key that distinguishes compiled knowledge units from ordinary chunks and// routes retrieval-side filters (e.g. "compile_kwd": ["artifact_page"]).var variantCompileKWD = map[common.Variant]string{- common.VariantStructure: "structure",- common.VariantWiki: "artifact_page",- common.VariantTree: "tree",- common.VariantMindmap: "mindmap",- common.VariantDatasetnav: "dataset_nav",+ common.VariantStructure: "structure",+ common.VariantWiki: "artifact_page",+ common.VariantTree: "tree",+ common.VariantMindmap: "mindmap",}// productsToChunkDocs converts the internal compiled Product rows into@@ -520,35 +676,6 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {return err}}-- case common.VariantDatasetnav:- // nav_cluster / nav_doc rows: type_kwd discriminates the row kind.- if v := metaString(p.Meta, "type"); v != "" {- if err := doc.SetExtraValue("type_kwd", v); err != nil {- return err- }- }- if v := metaString(p.Meta, "name"); v != "" {- if err := doc.SetExtraValue("title_kwd", v); err != nil {- return err- }- setTitleTokens(doc, v)- }- if v, ok := metaInt(p.Meta, "depth"); ok {- if err := doc.SetExtraValue("depth_int", v); err != nil {- return err- }- }- if v, ok := metaInt(p.Meta, "size"); ok {- if err := doc.SetExtraValue("doc_count_int", v); err != nil {- return err- }- }- if v := metaStringSlice(p.Meta, "doc_ids"); len(v) > 0 {- if err := doc.SetExtraValue("doc_ids_kwd", v); err != nil {- return err- }- }}return nildiff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex 93ff663315..074d3a1e47 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -13,6 +13,7 @@ import ("testing""ragflow/internal/ingestion/component/knowledge_compiler/common"+ "ragflow/internal/service/nav")// mockChat answers the structure variant's three LLM call shapes under the@@ -110,7 +111,7 @@ func (mockChat) Chat(_ context.Context, req common.ChatRequest) (*common.ChatRes}// proseChat returns generic, non-empty prose for the non-structure variants-// (wiki/tree/mindmap/datasetnav), which all just need a summary/outline text.+// (wiki/tree/mindmap), which all just need a summary/outline text.type proseChat struct{}func (proseChat) Chat(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {@@ -239,6 +240,138 @@ func TestKnowledgeCompiler_UnknownVariant(t *testing.T) {}}+// TestKnowledgeCompiler_Datasetnav_NoVariant locks the removal of the standalone+// datasetnav variant: dataset navigation is NOT an independent compile kind in+// Python (it is a by-product written after tree/page_index compile via+// internal/service datasetnav), so "datasetnav"/"dataset_nav" must now fail as+// an unknown variant rather than silently compile nothing.+func TestKnowledgeCompiler_Datasetnav_NoVariant(t *testing.T) {+ for _, kind := range []string{"datasetnav", "dataset_nav"} {+ _, err := common.KindToVariant(kind)+ if !errors.Is(err, common.ErrUnknownVariant) {+ t.Fatalf("KindToVariant(%q) err = %v, want ErrUnknownVariant", kind, err)+ }+ }+}++// fakeNavSvc records the most recent UpsertDoc call so a test can assert the+// tree by-product hook feeds the nav service the root document summary.+type fakeNavSvc struct {+ mu sync.Mutex+ called bool+ summary string+ docID string+ kbID string+}++func (f *fakeNavSvc) UpsertDoc(_ context.Context, in nav.UpsertDocInput) error {+ f.mu.Lock()+ defer f.mu.Unlock()+ f.called = true+ f.summary = in.Summary+ f.docID = in.DocID+ f.kbID = in.KbID+ return nil+}+func (f *fakeNavSvc) RemoveDoc(context.Context, string, string, string) error { return nil }+func (f *fakeNavSvc) Search(context.Context, string, string, string, []float32, int) ([]nav.NavHit, error) {+ return nil, nil+}+func (f *fakeNavSvc) ListClusters(context.Context, string, string, int, int) ([]nav.NavNode, int64, error) {+ return nil, 0, nil+}+func (f *fakeNavSvc) ListChildren(context.Context, string, string, string, int, int) ([]nav.NavNode, int64, error) {+ return nil, 0, nil+}++// TestKnowledgeCompiler_TreeNavByProduct asserts the tree compile-complete hook+// writes the dataset-nav by-product: after tree.Run produces a root product,+// upsertTreeNav calls NavService.UpsertDoc with the root summary and the+// doc/dataset context, and never aborts on failure.+func TestKnowledgeCompiler_TreeNavByProduct(t *testing.T) {+ fake := &fakeNavSvc{}+ nav.SetNavService(fake)+ t.Cleanup(func() { nav.SetNavService(nil) })++ deps := common.Deps{+ TenantID: "t1",+ DatasetID: "kb1",+ Chat: proseChat{},+ Embed: mockEmbedder{dim: 8},+ }+ products := []common.Product{+ {Content: "section one body", Meta: map[string]any{"kind": "summary", "level": 0}},+ {Content: "overall doc theme root summary", Vector: []float32{0.1, 0.2}, Meta: map[string]any{"kind": "root", "level": -1}},+ }+ upsertTreeNav(context.Background(), deps, "d1", products)++ fake.mu.Lock()+ defer fake.mu.Unlock()+ if !fake.called {+ t.Fatal("upsertTreeNav did not call NavService.UpsertDoc")+ }+ if fake.docID != "d1" {+ t.Errorf("doc_id = %q, want d1", fake.docID)+ }+ if fake.kbID != "kb1" {+ t.Errorf("kb_id = %q, want kb1", fake.kbID)+ }+ if !strings.Contains(fake.summary, "overall doc theme root summary") {+ t.Errorf("summary = %q, want the tree root summary", fake.summary)+ }+}++// TestKnowledgeCompiler_TreeNavByProduct_NilServiceDoesNotAbort asserts the+// by-product hook tolerates a missing NavService without erroring (nav is a+// derived artifact; its absence must never abort compilation).+func TestKnowledgeCompiler_TreeNavByProduct_NilServiceDoesNotAbort(t *testing.T) {+ nav.SetNavService(nil)+ t.Cleanup(func() { nav.SetNavService(nil) })+ // Should not panic and must return normally.+ upsertTreeNav(context.Background(), common.Deps{}, "d1",+ []common.Product{{Content: "x", Meta: map[string]any{"kind": "root"}}})+}++// TestKnowledgeCompiler_StructureNavByProduct asserts the structure/page_index+// by-product hook summarizes the graph product entities and feeds NavService.+func TestKnowledgeCompiler_StructureNavByProduct(t *testing.T) {+ fake := &fakeNavSvc{}+ nav.SetNavService(fake)+ t.Cleanup(func() { nav.SetNavService(nil) })++ graphJSON := `{"entities":[{"name":"Engine","description":"a propulsion device"},{"name":"Fuel","description":"combustion source"}]}`+ products := []common.Product{+ {Content: graphJSON, Vector: []float32{0.1, 0.2}, Meta: map[string]any{"kind": "graph", "compile_kwd": "page_index"}},+ }+ // The by-product embeds the SUMMARY (not the graph JSON vector), so an+ // embedder must be present.+ upsertStructureNav(context.Background(), common.Deps{TenantID: "t1", DatasetID: "kb1", Embed: mockEmbedder{dim: 8}}, "d1", products)++ fake.mu.Lock()+ defer fake.mu.Unlock()+ if !fake.called {+ t.Fatal("upsertStructureNav did not call NavService.UpsertDoc")+ }+ if fake.docID != "d1" {+ t.Errorf("doc_id = %q, want d1", fake.docID)+ }+ if !strings.Contains(fake.summary, "Engine: a propulsion device") {+ t.Errorf("summary = %q, want entity descriptions", fake.summary)+ }+}++// TestPageIndexSummary asserts entity descriptions are concatenated into a+// document summary.+func TestPageIndexSummary(t *testing.T) {+ got := pageIndexSummary(`{"entities":[{"name":"A","description":"one thing"},{"name":"B","description":""}]}`)+ if !strings.Contains(got, "A: one thing") {+ t.Errorf("summary = %q, want entity A", got)+ }+ if strings.Contains(got, "B") {+ t.Errorf("summary should skip empty-description entity, got %q", got)+ }+}+func TestKnowledgeCompiler_Alias_Mindmap(t *testing.T) {installMockDeps(t)// "mind_map" is the deprecated alias for "mindmap"; both resolve to the@@ -388,20 +521,6 @@ func TestKnowledgeCompiler_Mindmap_EndToEnd(t *testing.T) {}}-func TestKnowledgeCompiler_Datasetnav_EndToEnd(t *testing.T) {- installProseDeps(t)- chunks := runVariant(t, "datasetnav", nil)- foundRoot := false- for _, c := range chunks {- if kind, _ := c["kc_kind"].(string); kind == "root" {- foundRoot = true- }- }- if !foundRoot {- t.Fatalf("datasetnav: no 'root' chunk; got %d chunks", len(chunks))- }-}-// TestKnowledgeCompiler_EmitsChunks verifies that after compiling, the// component returns the knowledge units merged into the chunk stream (no// separate products/writer seam). The output shape is the chunker's@@ -786,140 +905,6 @@ func TestKnowledgeCompiler_Wiki_HistoricalDedupScopedByDataset(t *testing.T) {}}-// fakeDatasetnavLock records the key it was asked to acquire, so a test can-// assert the datasetnav rebuild lock is scoped to the dataset, not the document.-type fakeDatasetnavLock struct {- mu sync.Mutex- lastKey string-}--func (f *fakeDatasetnavLock) Acquire(_ context.Context, key string) (bool, error) {- f.mu.Lock()- defer f.mu.Unlock()- f.lastKey = key- return true, nil-}-func (f *fakeDatasetnavLock) Release(_ context.Context, _ string) error { return nil }--// TestKnowledgeCompiler_Datasetnav_LockKeyedByDataset is a regression for the-// Medium bug: the distributed rebuild lock used to be keyed by doc_id, so two-// runs against the same dataset (with different per-document doc_id values)-// took different locks and could interleave. This test supplies both doc_id-// ("d1") and dataset_id ("ds1") and asserts the lock key uses the dataset.-func TestKnowledgeCompiler_Datasetnav_LockKeyedByDataset(t *testing.T) {- lk := &fakeDatasetnavLock{}- common.SetDepsResolver(func(tenantID, llmID, embeddingModel string) (common.Deps, error) {- return common.Deps{- Chat: proseChat{},- Embed: mockEmbedder{dim: 8},- TenantID: tenantID,- Redis: lk,- }, nil- })- t.Cleanup(func() { common.SetDepsResolver(nil) })-- installVariantTemplateResolver(t, "datasetnav")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{- "compilation_template_id": "tpl-datasetnav", "llm_id": "llm1", "embedding_model": "emb1",- })- if err != nil {- t.Fatalf("NewKnowledgeCompilerComponent: %v", err)- }- if _, err := c.Invoke(context.Background(), nil, map[string]any{- "chunks": []any{- map[string]any{"id": "c1", "text": "The quick brown fox jumps over the lazy dog near the river bank."},- map[string]any{"id": "c2", "text": "A red fox and a lazy dog rest beside a calm river at dawn."},- },- "doc_id": "d1",- "dataset_id": "ds1",- "tenant_id": "t1",- }); err != nil {- t.Fatalf("Invoke: %v", err)- }- lk.mu.Lock()- key := lk.lastKey- lk.mu.Unlock()- want := "datasetnav:t1:ds1"- if key != want {- t.Fatalf("datasetnav lock key = %q, want %q (lock must be scoped to the dataset, not the document)", key, want)- }-}--// recordingNavChat echoes each nav-group summary back verbatim (so the child-// summaries are identifiable) and records the root-synthesis user prompt so a-// test can assert which child summaries reached the root overview.-type recordingNavChat struct {- mu sync.Mutex- rootPrompt string-}--func (r *recordingNavChat) Chat(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {- if strings.HasPrefix(req.UserPrompt, "Compose a navigation overview") {- r.mu.Lock()- r.rootPrompt = req.UserPrompt- r.mu.Unlock()- return &common.ChatResponse{Content: "root overview"}, nil- }- // Nav-group summary: echo the group text so the marker survives into the- // root prompt when the root is built from all child summaries.- return &common.ChatResponse{Content: strings.TrimSpace(req.UserPrompt)}, nil-}--// TestKnowledgeCompiler_Datasetnav_RootIncludesAllSummaries is a regression for-// root-overview completeness: the root synthesis must be built from ALL child-// summaries, not from a partial buffer. The Go implementation buffers every-// product in Outputs.Products (no streaming sink), so the root always sees the-// full child set even when the result set is large. Here nav_radius>1 forces-// each chunk into its own nav node, and we assert every child marker appears in-// the recorded root-synthesis prompt.-func TestKnowledgeCompiler_Datasetnav_RootIncludesAllSummaries(t *testing.T) {- chat := &recordingNavChat{}- common.SetDepsResolver(func(tenantID, llmID, embeddingModel string) (common.Deps, error) {- return common.Deps{- Chat: chat,- Embed: mockEmbedder{dim: 8},- TenantID: tenantID,- }, nil- })- t.Cleanup(func() { common.SetDepsResolver(nil) })-- installVariantTemplateResolver(t, "datasetnav")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{- "compilation_template_id": "tpl-datasetnav", "llm_id": "llm1", "embedding_model": "emb1",- // nav_radius > 1 guarantees no two chunks group together (cosine <= 1),- // so each chunk becomes its own nav node.- "extra": map[string]any{"nav_radius": 1.1},- })- if err != nil {- t.Fatalf("NewKnowledgeCompilerComponent: %v", err)- }-- markers := []string{"NAVMARKA", "NAVMARKB", "NAVMARKC", "NAVMARKD", "NAVMARKE"}- chunks := make([]any, len(markers))- for i, m := range markers {- chunks[i] = map[string]any{"id": m, "text": m + " unique section body text"}- }- if _, err := c.Invoke(context.Background(), nil, map[string]any{- "chunks": chunks,- "doc_id": "d1",- "tenant_id": "t1",- }); err != nil {- t.Fatalf("Invoke: %v", err)- }-- chat.mu.Lock()- root := chat.rootPrompt- chat.mu.Unlock()- if root == "" {- t.Fatalf("root synthesis prompt was never recorded (root node not built)")- }- for _, m := range markers {- if !strings.Contains(root, m) {- t.Fatalf("root overview is missing child summary %q; the root must be built from ALL child summaries.\nroot prompt:\n%s", m, root)- }- }-}-// fencedChat wraps otherwise-valid extraction JSON in a ```json ... ``` fence,// the most common way LLMs wrap JSON even when JSONMode is requested.type fencedChat struct{}diff --git a/internal/ingestion/component/knowledge_compiler/datasetnav/datasetnav.go b/internal/ingestion/component/knowledge_compiler/datasetnav/datasetnav.godeleted file mode 100644index f604e7be4b..0000000000--- a/internal/ingestion/component/knowledge_compiler/datasetnav/datasetnav.go+++ /dev/null@@ -1,776 +0,0 @@-// Package datasetnav implements the "datasetnav" variant of KnowledgeCompiler,-// mirroring Python's dataset_nav.py incremental clustering: each input chunk-// (the in-run equivalent of a document leaf) is embedded and placed into the-// nearest nav_cluster via layered KNN descent — merged when the similarity-// clears _MERGE_THRESHOLD, given a fresh sibling cluster above _MIN_SIM, or-// given a new root-level cluster otherwise. Clusters that exceed the fanout-// or doc-count caps split via the same 2-means procedure as Python.-//-// The Go port keeps the whole tree in memory for one Invoke (no ES reads or-// writes): cross-run incremental upsert/removal, which in Python is an-// ES-backed read-modify-write, is the caller's concern. The per-run algorithm-// (thresholds, LLM merge/summary prompts, readable cluster names, split) is-// faithful to the Python original.-package datasetnav--import (- "context"- "encoding/json"- "fmt"- "math"- "strings"-- "ragflow/internal/ingestion/component/knowledge_compiler/common"-)--// Thresholds and caps, mirrored from dataset_nav.py.-const (- mergeThresholdDefault = 0.80 // merge doc into cluster- recurseThreshold = 0.65 // continue descending into children- minSim = 0.50 // minimum similarity to be considered related- maxFanout = 64 // max child count before triggering rebalance- maxDocsPerCluster = 50 // max docs per leaf cluster before triggering split-)--// datasetnavLock is the minimal locking surface used to serialize dataset-level-// rebuilds. The production Redis client implements this via SET NX + TTL; tests-// inject an in-memory lock (or nil for no-op).-type datasetnavLock interface {- Acquire(ctx context.Context, key string) (bool, error)- Release(ctx context.Context, key string) error-}--// navCluster is one internal tree node (Python's nav_cluster row).-type navCluster struct {- Name string- Desc string- Parent string // parent cluster name; "root" for depth-0 clusters- Depth int- DocIDs []string- Vector []float32-}--// navDoc is one document leaf (Python's nav_doc row).-type navDoc struct {- ChunkID string- Text string- Parent string // owning cluster name- Depth int- Vector []float32-}--// navChild is one child reference under a cluster.-type navChild struct {- name string // cluster name or doc chunk id- isDoc bool- vec []float32-}--// navTree holds the in-memory tree built over one Invoke.-type navTree struct {- clusters map[string]*navCluster- order []string // cluster names in creation order- docs []*navDoc- children map[string][]navChild // parent cluster name → children-}--func newNavTree() *navTree {- return &navTree{clusters: map[string]*navCluster{}, children: map[string][]navChild{}}-}--func (t *navTree) addCluster(c *navCluster) {- t.clusters[c.Name] = c- t.order = append(t.order, c.Name)-}--// Run executes the datasetnav variant.-func Run(ctx context.Context, deps common.Deps, param common.Param, inputs common.Inputs) (common.Outputs, error) {- if deps.Embed == nil {- return common.Outputs{}, fmt.Errorf("datasetnav: embedder required")- }- docID := firstNonEmpty(inputs.DocID, deps.DatasetID)- if docID == "" {- docID = "unknown"- }- llmID := firstNonEmpty(param.LLMID, inputs.LLMID)- tenantID := deps.TenantID-- // The rebuild lock is scoped to the dataset, not the document (mirrors- // Python's _nav_lock_key(kb_id)): concurrent runs against the same dataset- // must share one lock. Fall back to docID only when no dataset id is known.- datasetID := firstNonEmpty(deps.DatasetID, docID)-- texts, chunkIDs := chunkTexts(inputs.Chunks)- if len(texts) == 0 {- return common.Outputs{}, nil- }-- if l, ok := deps.Redis.(datasetnavLock); ok && l != nil {- lockKey := "datasetnav:" + tenantID + ":" + datasetID- acquired, err := l.Acquire(ctx, lockKey)- if err != nil {- return common.Outputs{}, err- }- if !acquired {- return common.Outputs{}, fmt.Errorf("datasetnav: lock not acquired for %s", lockKey)- }- defer l.Release(ctx, lockKey)- }-- // Embed every chunk text once (Python embeds each doc summary on upsert).- vectors, err := deps.Embed.Encode(ctx, texts)- if err != nil {- return common.Outputs{}, err- }- if len(vectors) != len(texts) {- return common.Outputs{}, fmt.Errorf("datasetnav: embedding count mismatch (%d vs %d)", len(vectors), len(texts))- }-- // Sequential placement, mirroring the lock-serialized per-document upserts- // in Python (also what makes the in-run tree deterministic).- tree := newNavTree()- var clusterDescs []string- mergeThreshold := mergeThresholdFor(param)- for i, text := range texts {- if err := ctx.Err(); err != nil {- return common.Outputs{}, err- }- desc, err := tree.place(ctx, deps, llmID, chunkIDs[i], text, vectors[i], mergeThreshold)- if err != nil {- return common.Outputs{}, err- }- clusterDescs = append(clusterDescs, desc...)- }-- // Emit cluster rows in creation order, then nav_doc leaves in placement- // order, then the root overview node (a Go-side convenience: Python's tree- // has no root row, but downstream consumers expect one overview product).- var products []common.Product- clusterProductID := map[string]string{}- // Pre-compute every cluster id (deterministic from StableRowID) before- // resolving parent edges, so a child emitted before its (later-created)- // parent still gets the correct ParentID instead of "" (reparenting bug).- // Cluster ids are tenant-scoped (as the nav_doc ids already are) so two- // tenants whose datasetID falls back to the same docID cannot collide.- for _, name := range tree.order {- clusterProductID[name] = common.StableRowID("dataset_nav", tenantID, datasetID, "cluster", name)- }- for _, name := range tree.order {- c := tree.clusters[name]- pid := ""- if c.Parent != "" && c.Parent != "root" {- pid = clusterProductID[c.Parent]- }- id := clusterProductID[c.Name]- products = append(products, common.Product{- ID: id,- DocID: docID,- TenantID: tenantID,- Variant: common.VariantDatasetnav,- Content: payloadJSON(map[string]any{"type": "nav_cluster", "description": c.Desc}),- Vector: c.Vector,- ParentID: pid,- Meta: map[string]any{- "kind": "nav_cluster",- "type": "nav_cluster",- "name": c.Name,- "parent_name": c.Parent,- "depth": c.Depth,- "doc_ids": append([]string{}, c.DocIDs...),- "size": len(c.DocIDs),- },- })- }- for _, d := range tree.docs {- products = append(products, common.Product{- // Scope nav_doc ids by tenant and dataset (not just chunk id) so- // synthetic/positional chunk ids from different docs/datasets do not- // collide and overwrite each other across tenants (M5).- ID: common.StableRowID("dataset_nav", tenantID, datasetID, "doc", d.ChunkID),- DocID: docID,- TenantID: tenantID,- Variant: common.VariantDatasetnav,- Content: payloadJSON(map[string]any{"type": "nav_doc", "description": d.Text}),- Vector: d.Vector,- ParentID: clusterProductID[d.Parent],- Meta: map[string]any{- "kind": "nav_doc",- "type": "nav_doc",- "name": navDocName(d.Parent, d.Text),- "parent_name": d.Parent,- "depth": d.Depth,- "doc_ids": []string{d.ChunkID},- },- })- }-- // Root overview built from EVERY cluster description (all products are- // buffered in the same slice, so every node contributes to the root).- rootSummary, err := summarize(ctx, deps, llmID, "Compose a navigation overview from these section summaries:\n\n"+formatNavSummaries(clusterDescs))- if err == nil && rootSummary != "" {- emb, e2 := deps.Embed.Encode(ctx, []string{rootSummary})- if e2 == nil && len(emb) > 0 {- products = append(products, common.Product{- ID: common.StableRowID(tenantID, docID, string(common.VariantDatasetnav), "root"),- DocID: docID,- TenantID: tenantID,- Variant: common.VariantDatasetnav,- Content: rootSummary,- Vector: emb[0],- Meta: map[string]any{- "kind": "root",- "type": "nav_cluster",- "name": "root",- "depth": 0,- "size": len(tree.docs),- },- })- }- }-- out := common.Outputs{- Products: products,- }- return out, nil-}--// place inserts one document (chunk) into the tree, mirroring-// upsert_dataset_nav_doc's placement logic. It returns the descriptions of-// any clusters created for this document (for the root overview).-func (t *navTree) place(ctx context.Context, deps common.Deps, llmID, chunkID, text string, vec []float32, mergeThreshold float64) ([]string, error) {- bestName, bestParent, sim := t.findBestCluster(vec)-- switch {- case bestName != "" && sim >= mergeThreshold:- // ── Merge into the best cluster (mirrors _llm_merge + re-embed) ──- c := t.clusters[bestName]- newDesc, err := llmMerge(ctx, deps, llmID, c.Desc, text)- if err != nil {- return nil, err- }- if newDesc != c.Desc {- c.Desc = newDesc- emb, err := deps.Embed.Encode(ctx, []string{newDesc})- if err != nil {- return nil, err- }- if len(emb) > 0 {- c.Vector = emb[0]- }- }- if !containsString(c.DocIDs, chunkID) {- c.DocIDs = append(c.DocIDs, chunkID)- }- t.addDoc(c, chunkID, text, vec, c.Depth+1)- if err := t.maybeSplit(ctx, deps, llmID, bestName); err != nil {- return nil, err- }- return nil, nil-- case bestName != "" && sim >= minSim:- // ── Create a sibling/child cluster (mirrors the _MIN_SIM branch) ──- parentForNew := bestParent- if parentForNew == "" {- parentForNew = bestName- }- parentDepth := 1 // Python's default when the parent row is absent- if p, ok := t.clusters[parentForNew]; ok {- parentDepth = p.Depth- }- title, desc, err := llmCreateSummary(ctx, deps, llmID, []string{text})- if err != nil {- return nil, err- }- name := readableClusterName(title, text)- nc := &navCluster{- Name: name,- Desc: desc,- Parent: parentForNew,- // Python stamps the new cluster at the PARENT's depth (not +1);- // the nav_doc goes one deeper. Mirrored, quirk included.- Depth: parentDepth,- DocIDs: []string{chunkID},- Vector: vec,- }- t.addCluster(nc)- t.children[parentForNew] = append(t.children[parentForNew], navChild{name: name, vec: vec})- t.addDoc(nc, chunkID, text, vec, parentDepth+1)- return []string{desc}, nil-- default:- // ── Create a root-level cluster (mirrors the else branch) ──- title, desc, err := llmCreateSummary(ctx, deps, llmID, []string{text})- if err != nil {- return nil, err- }- name := readableClusterName(title, text)- nc := &navCluster{- Name: name,- Desc: desc,- Parent: "root",- Depth: 0,- DocIDs: []string{chunkID},- Vector: vec,- }- t.addCluster(nc)- t.addDoc(nc, chunkID, text, vec, 1)- return []string{desc}, nil- }-}--func (t *navTree) addDoc(c *navCluster, chunkID, text string, vec []float32, depth int) {- t.docs = append(t.docs, &navDoc{ChunkID: chunkID, Text: text, Parent: c.Name, Depth: depth, Vector: vec})- t.children[c.Name] = append(t.children[c.Name], navChild{name: chunkID, isDoc: true, vec: vec})-}--// findBestCluster mirrors _find_best_cluster: top-1 root cluster by cosine,-// then descend into the best-matching child while similarity stays above-// _RECURSE_THRESHOLD. Returns (best cluster name, its parent name, sim).-func (t *navTree) findBestCluster(vec []float32) (string, string, float64) {- var best *navCluster- bestSim := 0.0- for _, c := range t.clusters {- if c.Depth != 0 {- continue- }- s := cosine(vec, c.Vector)- if best == nil || s > bestSim {- best, bestSim = c, s- }- }- if best == nil {- return "", "", 0.0- }- bestName, bestParent, sim := best.Name, best.Parent, bestSim- visited := map[string]bool{bestName: true}- for sim >= recurseThreshold {- var child *navCluster- childSim := 0.0- for _, k := range t.children[bestName] {- if k.isDoc || visited[k.name] {- continue- }- c := t.clusters[k.name]- if c == nil {- continue- }- if s := cosine(vec, c.Vector); child == nil || s > childSim {- child, childSim = c, s- }- }- if child == nil || childSim < recurseThreshold {- break- }- bestParent = best.Parent // the (old) best's parent, mirroring Python- bestName = child.Name- sim = childSim- best = child- visited[bestName] = true- }- return bestName, bestParent, sim-}--// maybeSplit mirrors _maybe_split_cluster: when a cluster exceeds the fanout-// or doc-count caps, split its children into two 2-means groups and reparent-// each group under a fresh sub-cluster.-func (t *navTree) maybeSplit(ctx context.Context, deps common.Deps, llmID, clusterName string) error {- kids := t.children[clusterName]- var clusterKids, docKids []navChild- for _, k := range kids {- if k.isDoc {- docKids = append(docKids, k)- } else {- clusterKids = append(clusterKids, k)- }- }- if len(clusterKids)+len(docKids) <= maxFanout && len(docKids) <= maxDocsPerCluster {- return nil- }- if len(kids) < 4 {- return nil // Python requires >= 4 embeddings to attempt a split- }-- embs := make([][]float32, len(kids))- for i, k := range kids {- embs[i] = k.vec- }- labels := twoMeans(embs)-- parent := t.clusters[clusterName]- depth := 0- if parent != nil {- depth = parent.Depth + 1- }- // navChild collects the new sub-cluster names so the split parent keeps a- // children entry pointing at them; without it findBestCluster can no longer- // descend into the split subtree (M3).- var subChildren []navChild- for gi := 0; gi < 2; gi++ {- var kidIdx []int- for i, lb := range labels {- if lb == gi {- kidIdx = append(kidIdx, i)- }- }- if len(kidIdx) == 0 {- continue- }- var docIDs, descs []string- for _, i := range kidIdx {- k := kids[i]- if k.isDoc {- if !containsString(docIDs, k.name) {- docIDs = append(docIDs, k.name)- }- for _, d := range t.docs {- if d.ChunkID == k.name {- descs = append(descs, d.Text)- }- }- continue- }- if c := t.clusters[k.name]; c != nil {- for _, d := range c.DocIDs {- if !containsString(docIDs, d) {- docIDs = append(docIDs, d)- }- }- descs = append(descs, c.Desc)- }- }- var title, desc string- if len(descs) > 0 {- var err error- title, desc, err = llmCreateSummary(ctx, deps, llmID, descs)- if err != nil {- return err- }- } else {- title, desc = fmt.Sprintf("Group %d", gi+1), fmt.Sprintf("Group %d", gi+1)- }- gname := readableClusterName(title, desc)- emb, err := deps.Embed.Encode(ctx, []string{desc})- if err != nil {- return err- }- var gvec []float32- if len(emb) > 0 {- gvec = emb[0]- }- nc := &navCluster{Name: gname, Desc: desc, Parent: clusterName, Depth: depth, DocIDs: docIDs, Vector: gvec}- t.addCluster(nc)- subChildren = append(subChildren, navChild{name: gname})-- // Reparent the group's children to the new sub-cluster.- for _, i := range kidIdx {- k := kids[i]- if k.isDoc {- for _, d := range t.docs {- if d.ChunkID == k.name {- d.Parent = gname- d.Depth = depth + 1- }- }- } else if c := t.clusters[k.name]; c != nil {- c.Parent = gname- c.Depth = depth + 1- }- t.children[gname] = append(t.children[gname], k)- }- }- // The split parent's children become the new sub-clusters so findBestCluster- // can still descend into the split subtree (M3). When no sub-cluster was- // actually created the parent keeps an empty children list.- t.children[clusterName] = subChildren- return nil-}--// twoMeans mirrors the runtime k-means-like split in _maybe_split_cluster:-// centroids seeded with the first and middle embeddings, 10 refinement-// rounds, squared-euclidean assignment, then a final relabel pass.-func twoMeans(embs [][]float32) []int {- n := len(embs)- if n == 0 {- return nil- }- centroids := [][]float32{append([]float32{}, embs[0]...), append([]float32{}, embs[n/2]...)}- assign := func(e []float32) int {- if sqDist(e, centroids[0]) < sqDist(e, centroids[1]) {- return 0- }- return 1- }- for iter := 0; iter < 10; iter++ {- var groups [2][][]float32- for _, e := range embs {- g := assign(e)- groups[g] = append(groups[g], e)- }- for gi := 0; gi < 2; gi++ {- if len(groups[gi]) == 0 {- continue- }- dim := len(groups[gi][0])- avg := make([]float32, dim)- for _, e := range groups[gi] {- for d := 0; d < dim; d++ {- avg[d] += e[d]- }- }- for d := range avg {- avg[d] /= float32(len(groups[gi]))- }- centroids[gi] = avg- }- }- labels := make([]int, n)- for i, e := range embs {- labels[i] = assign(e)- }- return labels-}--func sqDist(a, b []float32) float64 {- var s float64- for i := 0; i < len(a) && i < len(b); i++ {- d := float64(a[i] - b[i])- s += d * d- }- return s-}--func cosine(a, b []float32) float64 {- if len(a) == 0 || len(b) == 0 || len(a) != len(b) {- return 0- }- var dot, na, nb float64- for i := range a {- dot += float64(a[i]) * float64(b[i])- na += float64(a[i]) * float64(a[i])- nb += float64(b[i]) * float64(b[i])- }- if na == 0 || nb == 0 {- return 0- }- return dot / (math.Sqrt(na) * math.Sqrt(nb))-}--// ---- LLM helpers (prompts mirrored verbatim from dataset_nav.py) ------var navLLMTemperature = 0.1--// llmMerge mirrors _llm_merge: fuse the existing cluster description with the-// new doc summary. The reply may be a JSON object ({"merged"|"result"}) or-// bare text; anything unusable keeps the old description.-func llmMerge(ctx context.Context, deps common.Deps, llmID, clusterDesc, docSummary string) (string, error) {- if deps.Chat == nil {- return clusterDesc, nil- }- prompt := "Merge the following two descriptions of the same topic into " +- "a single concise summary (1-3 sentences):\n\n" +- "Existing: " + clusterDesc + "\n\n" +- "New: " + docSummary + "\n\n" +- "Return ONLY the merged text, no commentary."- resp, err := deps.Chat.Chat(ctx, common.ChatRequest{LLMID: llmID, UserPrompt: prompt, Temperature: &navLLMTemperature})- if err != nil {- return "", err- }- text := strings.TrimSpace(resp.Content)- if m := parseWholeJSON(text); m != nil {- if s, ok := m["merged"].(string); ok && strings.TrimSpace(s) != "" {- return s, nil- }- if s, ok := m["result"].(string); ok && strings.TrimSpace(s) != "" {- return s, nil- }- return clusterDesc, nil- }- if text != "" {- return text, nil- }- return clusterDesc, nil-}--// llmCreateSummary mirrors _llm_create_summary: derive a readable (name,-// summary) pair from doc summaries, with the same fallbacks.-func llmCreateSummary(ctx context.Context, deps common.Deps, llmID string, docSummaries []string) (string, string, error) {- fallbackSummary := ""- if len(docSummaries) > 0 {- fallbackSummary = docSummaries[0]- }- if deps.Chat == nil {- return fallbackTitle(fallbackSummary), fallbackSummary, nil- }- prompt := "Given the document excerpts below, produce a short human-readable topic " +- "name and a concise description of their common topic.\n\n" +- strings.Join(docSummaries, "\n---\n") + "\n\n" +- `Return ONLY JSON: {"name": "<2-6 word topic title>", "summary": "<1-3 sentence description>"}`- resp, err := deps.Chat.Chat(ctx, common.ChatRequest{LLMID: llmID, UserPrompt: prompt, Temperature: &navLLMTemperature})- if err != nil {- return "", "", err- }- text := strings.TrimSpace(resp.Content)- if m := parseWholeJSON(text); m != nil {- summary := fallbackSummary- if s, ok := m["summary"].(string); ok && strings.TrimSpace(s) != "" {- summary = strings.TrimSpace(s)- } else if s, ok := m["result"].(string); ok && strings.TrimSpace(s) != "" {- summary = strings.TrimSpace(s)- }- name := cleanTitle(firstStringOf(m["name"]))- if name == "" {- name = fallbackTitle(summary)- }- return name, summary, nil- }- if text != "" {- return fallbackTitle(text), text, nil- }- return fallbackTitle(fallbackSummary), fallbackSummary, nil-}--// parseWholeJSON parses the response ONLY when it is a complete JSON object-// (mirrors gen_json's dict path; bare text takes the string fallback path).-func parseWholeJSON(s string) map[string]any {- if !strings.HasPrefix(s, "{") {- return nil- }- var m map[string]any- if err := json.Unmarshal([]byte(s), &m); err != nil {- return nil- }- return m-}--// cleanTitle mirrors _clean_title: whitespace-normalized, capped at 48 chars.-func cleanTitle(title string) string {- return truncateRunes(strings.Join(strings.Fields(title), " "), 48)-}--// fallbackTitle mirrors _fallback_title: first 6 words, else "Cluster".-func fallbackTitle(summary string) string {- words := strings.Fields(summary)- if len(words) > 6 {- words = words[:6]- }- if t := strings.Join(words, " "); t != "" {- return t- }- return "Cluster"-}--// readableClusterName mirrors _readable_cluster_name: "<title> <8-hex>".-func readableClusterName(title, seed string) string {- t := cleanTitle(title)- if t == "" {- t = "Cluster"- }- return t + " " + common.ContentHash(seed)[:8]-}--// navDocName mirrors _make_nav_doc_row's name field:-// f"{parent_kwd}_{xxh64(summary)[:12]}".-func navDocName(parent, summary string) string {- return parent + "_" + common.ContentHash(summary)[:12]-}--// summarize is the plain-text LLM helper used for the root overview.-func summarize(ctx context.Context, deps common.Deps, llmID, text string) (string, error) {- if deps.Chat == nil {- return "", nil- }- resp, err := deps.Chat.Chat(ctx, common.ChatRequest{- LLMID: llmID,- SystemPrompt: navSystemPrompt,- UserPrompt: text,- })- if err != nil {- return "", err- }- return strings.TrimSpace(resp.Content), nil-}--// formatNavSummaries renders cluster descriptions as a bullet list for the-// root overview prompt.-func formatNavSummaries(summaries []string) string {- var b strings.Builder- for _, s := range summaries {- b.WriteString("- ")- b.WriteString(s)- b.WriteString("\n")- }- return b.String()-}--// mergeThresholdFor returns the merge threshold (nav_radius override kept for-// tuning/tests; default _MERGE_THRESHOLD).-func mergeThresholdFor(param common.Param) float64 {- if t, ok := param.Extra["nav_radius"].(float64); ok && t > 0 {- return t- }- return mergeThresholdDefault-}--// chunkTexts returns the non-empty chunk texts and their ids in parallel-// order (Python skips docs without a summary; we skip empty chunks).-func chunkTexts(chunks []common.Chunk) ([]string, []string) {- var texts, ids []string- for i, c := range chunks {- t := firstNonEmpty(c.Text, c.Content)- if strings.TrimSpace(t) == "" {- continue- }- id := c.ID- if id == "" {- id = fmt.Sprintf("chunk-%d", i)- }- texts = append(texts, t)- ids = append(ids, id)- }- return texts, ids-}--func payloadJSON(v map[string]any) string {- var b strings.Builder- enc := json.NewEncoder(&b)- enc.SetEscapeHTML(false)- if err := enc.Encode(v); err != nil {- return "{}"- }- return strings.TrimSpace(b.String())-}--func truncateRunes(s string, n int) string {- r := []rune(s)- if len(r) <= n {- return s- }- return string(r[:n])-}--func firstStringOf(v any) string {- if s, ok := v.(string); ok {- return s- }- return ""-}--func containsString(haystack []string, needle string) bool {- for _, s := range haystack {- if s == needle {- return true- }- }- return false-}--func firstNonEmpty(vals ...string) string {- for _, v := range vals {- if v != "" {- return v- }- }- return ""-}--const navSystemPrompt = `You are a navigation assistant. Summarize the provided text into a concise label and overview that helps a user navigate to the relevant content. Output the summary only.`diff --git a/internal/ingestion/component/knowledge_compiler/datasetnav/datasetnav_test.go b/internal/ingestion/component/knowledge_compiler/datasetnav/datasetnav_test.godeleted file mode 100644index 848e45f607..0000000000--- a/internal/ingestion/component/knowledge_compiler/datasetnav/datasetnav_test.go+++ /dev/null@@ -1,289 +0,0 @@-package datasetnav--import (- "context"- "fmt"- "strings"- "testing"-- "ragflow/internal/ingestion/component/knowledge_compiler/common"-)--// scriptedEmbedder maps texts to fixed vectors (fallback: halves).-type scriptedEmbedder struct {- vecs map[string][]float32-}--func (s scriptedEmbedder) Dimensions() int { return 2 }-func (s scriptedEmbedder) Encode(_ context.Context, texts []string) ([][]float32, error) {- out := make([][]float32, len(texts))- for i, t := range texts {- if v, ok := s.vecs[t]; ok {- out[i] = v- } else {- out[i] = []float32{0.5, 0.5}- }- }- return out, nil-}--// navFakeChat answers the three nav LLM calls deterministically: merge keeps-// the existing description, create-summary returns canned JSON, the root-// overview is a fixed string.-type navFakeChat struct {- mergeCalls int- summaryCalls int-}--func (m *navFakeChat) Chat(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {- switch {- case strings.Contains(req.UserPrompt, "Merge the following two descriptions"):- m.mergeCalls++- existing := ""- if rest, ok := strings.CutPrefix(req.UserPrompt, "Merge the following two descriptions of the same topic into a single concise summary (1-3 sentences):\n\nExisting: "); ok {- existing, _, _ = strings.Cut(rest, "\n\nNew: ")- }- return &common.ChatResponse{Content: fmt.Sprintf(`{"merged": %q}`, existing)}, nil- case strings.Contains(req.UserPrompt, "Given the document excerpts below"):- m.summaryCalls++- // Distinct summaries per call: readable cluster names hash the- // description, so identical canned summaries would collide (the same- // name-overwrite hazard exists in Python's _nav_cluster_id).- return &common.ChatResponse{Content: fmt.Sprintf(`{"name": "Topic", "summary": "a topic summary %d"}`, m.summaryCalls)}, nil- case strings.HasPrefix(req.UserPrompt, "Compose a navigation overview"):- return &common.ChatResponse{Content: "root overview"}, nil- default:- return &common.ChatResponse{Content: "ok"}, nil- }-}--func TestTwoMeans(t *testing.T) {- embs := [][]float32{{1, 0}, {0.9, 0.1}, {0, 1}, {0.1, 0.9}}- labels := twoMeans(embs)- if len(labels) != 4 {- t.Fatalf("labels = %v", labels)- }- if labels[0] != labels[1] || labels[2] != labels[3] || labels[0] == labels[2] {- t.Fatalf("expected {x,x,y,y} grouping, got %v", labels)- }-}--func TestReadableClusterName(t *testing.T) {- got := readableClusterName(" My Topic ", "seed text")- if !strings.HasPrefix(got, "My Topic ") {- t.Fatalf("readableClusterName = %q, want cleaned title + hash suffix", got)- }- if got := readableClusterName("", "seed"); !strings.HasPrefix(got, "Cluster ") {- t.Fatalf("empty title must fall back to Cluster: %q", got)- }-}--func TestFallbackTitle(t *testing.T) {- if got := fallbackTitle("one two three four five six seven eight"); got != "one two three four five six" {- t.Fatalf("fallbackTitle = %q, want first 6 words", got)- }- if got := fallbackTitle(""); got != "Cluster" {- t.Fatalf("empty fallbackTitle = %q, want Cluster", got)- }-}--func TestPlace_MergeSiblingRootBranches(t *testing.T) {- vecs := map[string][]float32{- "A": {1, 0},- "A2": {0.99, 0.1}, // cosine ≈ 0.995 with A → merge- "B": {0.7, 0.714}, // cosine ≈ 0.7 with A → sibling- "C": {0, 1}, // cosine 0 with A → new root- }- deps := common.Deps{Chat: &navFakeChat{}, Embed: scriptedEmbedder{vecs: vecs}}- tree := newNavTree()-- if _, err := tree.place(context.Background(), deps, "llm", "cA", "A", vecs["A"], mergeThresholdDefault); err != nil {- t.Fatal(err)- }- if _, err := tree.place(context.Background(), deps, "llm", "cA2", "A2", vecs["A2"], mergeThresholdDefault); err != nil {- t.Fatal(err)- }- if _, err := tree.place(context.Background(), deps, "llm", "cB", "B", vecs["B"], mergeThresholdDefault); err != nil {- t.Fatal(err)- }- if _, err := tree.place(context.Background(), deps, "llm", "cC", "C", vecs["C"], mergeThresholdDefault); err != nil {- t.Fatal(err)- }-- if len(tree.order) != 3 {- t.Fatalf("clusters = %d, want 3 (A-root, B-sibling, C-root): %v", len(tree.order), tree.order)- }- // A2 merged into A's cluster.- var clusterA *navCluster- for _, name := range tree.order {- if tree.clusters[name].Depth == 0 && containsString(tree.clusters[name].DocIDs, "cA") {- clusterA = tree.clusters[name]- }- }- if clusterA == nil || len(clusterA.DocIDs) != 2 || clusterA.DocIDs[1] != "cA2" {- t.Fatalf("A2 must merge into cluster A (doc_ids=%v)", clusterA)- }- // B's cluster: sibling under the virtual root (Python depth quirk: parent- // depth default 1 → cluster depth 1, parent "root").- var clusterB *navCluster- for _, name := range tree.order {- c := tree.clusters[name]- if containsString(c.DocIDs, "cB") {- clusterB = c- }- }- if clusterB == nil || clusterB.Parent != "root" || clusterB.Depth != 1 {- t.Fatalf("B sibling cluster = %+v, want parent root depth 1", clusterB)- }- // C's cluster: a fresh root-level cluster.- var clusterC *navCluster- for _, name := range tree.order {- c := tree.clusters[name]- if containsString(c.DocIDs, "cC") {- clusterC = c- }- }- if clusterC == nil || clusterC.Parent != "root" || clusterC.Depth != 0 {- t.Fatalf("C root cluster = %+v, want parent root depth 0", clusterC)- }- if len(tree.docs) != 4 {- t.Fatalf("nav docs = %d, want 4", len(tree.docs))- }-}--func TestMaybeSplit(t *testing.T) {- vecs := map[string][]float32{}- deps := common.Deps{Chat: &navFakeChat{}, Embed: scriptedEmbedder{vecs: vecs}}- tree := newNavTree()- parent := &navCluster{Name: "P", Desc: "parent", Parent: "root", Depth: 0, Vector: []float32{1, 0}}- tree.addCluster(parent)- // 51 docs (> maxDocsPerCluster) alternating between two axes.- for i := 0; i < 51; i++ {- id := fmt.Sprintf("d%d", i)- v := []float32{1, 0}- if i%2 == 1 {- v = []float32{0, 1}- }- parent.DocIDs = append(parent.DocIDs, id)- tree.addDoc(parent, id, "text "+id, v, 1)- }- if err := tree.maybeSplit(context.Background(), deps, "llm", "P"); err != nil {- t.Fatal(err)- }- // Expect two new sub-clusters under P, children reparented, P left with- // no direct children.- if len(tree.order) != 3 {- t.Fatalf("clusters after split = %d, want 3 (P + 2 groups): %v", len(tree.order), tree.order)- }- // The split parent must now point at exactly the two new sub-clusters so- // findBestCluster can descend into the split subtree (M3).- if got := len(tree.children["P"]); got != 2 {- t.Fatalf("split parent children = %d, want 2 (the two new sub-clusters)", got)- }- subDocs := 0- for _, name := range tree.order[1:] {- c := tree.clusters[name]- if c.Parent != "P" || c.Depth != 1 {- t.Errorf("sub-cluster %s parent/depth = %s/%d, want P/1", name, c.Parent, c.Depth)- }- subDocs += len(c.DocIDs)- }- if subDocs != 51 {- t.Errorf("sub-cluster doc_ids total = %d, want 51", subDocs)- }- for _, d := range tree.docs {- if d.Parent == "P" {- t.Errorf("doc %s not reparented off P", d.ChunkID)- }- if d.Depth != 2 {- t.Errorf("doc %s depth = %d, want 2", d.ChunkID, d.Depth)- }- }-}--func TestMaybeSplit_BelowCaps(t *testing.T) {- tree := newNavTree()- parent := &navCluster{Name: "P", Desc: "p", Parent: "root", Depth: 0, Vector: []float32{1, 0}}- tree.addCluster(parent)- for i := 0; i < 3; i++ {- tree.addDoc(parent, fmt.Sprintf("d%d", i), "t", []float32{1, 0}, 1)- }- if err := tree.maybeSplit(context.Background(), common.Deps{}, "llm", "P"); err != nil {- t.Fatal(err)- }- if len(tree.order) != 1 {- t.Fatalf("below caps must not split: %v", tree.order)- }-}--func TestRun_EndToEnd(t *testing.T) {- vecs := map[string][]float32{- "alpha one": {1, 0},- "alpha two": {0.99, 0.1},- "beta one": {0, 1},- }- deps := common.Deps{Chat: &navFakeChat{}, Embed: scriptedEmbedder{vecs: vecs}, TenantID: "t1", DatasetID: "ds1"}- p := common.Param{}.Defaults()- p.Variant = common.VariantDatasetnav- inputs := common.Inputs{- DocID: "d1",- Chunks: []common.Chunk{- {ID: "c1", Text: "alpha one"},- {ID: "c2", Text: "alpha two"},- {ID: "c3", Text: "beta one"},- },- }- out, err := Run(context.Background(), deps, p, inputs)- if err != nil {- t.Fatalf("Run: %v", err)- }- var clusters, docs, roots int- parentByID := map[string]string{}- for _, p := range out.Products {- switch p.Meta["kind"] {- case "nav_cluster":- clusters++- parentByID[p.Meta["name"].(string)] = p.Meta["parent_name"].(string)- case "nav_doc":- docs++- if p.ParentID == "" {- t.Errorf("nav_doc must link to its cluster product")- }- case "root":- roots++- }- }- if clusters != 2 || docs != 3 || roots != 1 {- t.Fatalf("products = %d clusters + %d docs + %d roots, want 2+3+1", clusters, docs, roots)- }-}--func TestLLMMergeFallbacks(t *testing.T) {- // Non-JSON text reply is used verbatim (mirrors gen_json's str path).- deps := common.Deps{Chat: chatFunc(func(req common.ChatRequest) (*common.ChatResponse, error) {- return &common.ChatResponse{Content: "merged text"}, nil- })}- got, err := llmMerge(context.Background(), deps, "llm", "old", "new")- if err != nil || got != "merged text" {- t.Fatalf("llmMerge text path = %q, %v", got, err)- }- // Empty reply keeps the old description.- deps2 := common.Deps{Chat: chatFunc(func(req common.ChatRequest) (*common.ChatResponse, error) {- return &common.ChatResponse{Content: ""}, nil- })}- got, _ = llmMerge(context.Background(), deps2, "llm", "old", "new")- if got != "old" {- t.Fatalf("empty reply must keep old desc, got %q", got)- }- // Nil chat keeps the old description without a call.- got, _ = llmMerge(context.Background(), common.Deps{}, "llm", "old", "new")- if got != "old" {- t.Fatalf("nil chat must keep old desc, got %q", got)- }-}--type chatFunc func(common.ChatRequest) (*common.ChatResponse, error)--func (f chatFunc) Chat(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {- return f(req)-}diff --git a/internal/service/dataset_artifact_nav_test.go b/internal/service/dataset_artifact_nav_test.gonew file mode 100644index 0000000000..88c1746f41--- /dev/null+++ b/internal/service/dataset_artifact_nav_test.go@@ -0,0 +1,92 @@+package service++import (+ "context"+ "testing"++ "ragflow/internal/service/nav"+)++// fakeArtifactNav is an in-memory nav.NavService used to lock the limited+// DeleteNav/DeleteNavNode semantics (direct doc children only; no sub-cluster+// or cascade deletion).+type fakeArtifactNav struct {+ clusters []nav.NavNode+ children map[string][]nav.NavNode+ removedDocs []string+}++func (f *fakeArtifactNav) UpsertDoc(context.Context, nav.UpsertDocInput) error { return nil }+func (f *fakeArtifactNav) RemoveDoc(_ context.Context, _, _, docID string) error {+ f.removedDocs = append(f.removedDocs, docID)+ return nil+}+func (f *fakeArtifactNav) Search(context.Context, string, string, string, []float32, int) ([]nav.NavHit, error) {+ return nil, nil+}+func (f *fakeArtifactNav) ListClusters(context.Context, string, string, int, int) ([]nav.NavNode, int64, error) {+ return f.clusters, int64(len(f.clusters)), nil+}+func (f *fakeArtifactNav) ListChildren(_ context.Context, _, _, name string, _, _ int) ([]nav.NavNode, int64, error) {+ return f.children[name], int64(len(f.children[name])), nil+}++// TestDeleteNav_RemovesOnlyDirectDocChildren documents the limited semantic of+// the deprecated DeleteNav: it removes the nav_doc rows directly under root+// clusters, and does NOT touch sub-clusters (Python's full cascade is not+// implemented in the minimal loop).+func TestDeleteNav_RemovesOnlyDirectDocChildren(t *testing.T) {+ fake := &fakeArtifactNav{+ clusters: []nav.NavNode{{Name: "C1", Description: "root"}},+ children: map[string][]nav.NavNode{+ "C1": {{Name: "Sub", Type: "cluster"}, {Name: "DocA", Type: "doc", DocID: "d1"}},+ "Sub": {{Name: "DocB", Type: "doc", DocID: "d2"}},+ },+ }+ prev := nav.GetNavService()+ nav.SetNavService(fake)+ defer func() { nav.SetNavService(prev) }()++ svc := NewDatasetArtifactService()+ n, err := svc.DeleteNav(context.Background(), "t1", "kb1")+ if err != nil {+ t.Fatalf("DeleteNav: %v", err)+ }+ // Only the direct doc child (d1) under the root cluster is removed; the+ // sub-cluster's doc (d2) is NOT reached (no subtree traversal).+ if n != 1 {+ t.Errorf("deleted = %d, want 1 (direct doc child only)", n)+ }+ if len(fake.removedDocs) != 1 || fake.removedDocs[0] != "d1" {+ t.Errorf("removed docs = %v, want [d1] only (no cascade into Sub)", fake.removedDocs)+ }+}++// TestDeleteNavNode_RemovesDirectChildren documents that DeleteNavNode drains+// only the immediate doc children of a named cluster.+func TestDeleteNavNode_RemovesDirectChildren(t *testing.T) {+ fake := &fakeArtifactNav{+ children: map[string][]nav.NavNode{+ "C1": {+ {Name: "Sub2", Type: "cluster"},+ {Name: "DocA", Type: "doc", DocID: "d1"},+ {Name: "DocB", Type: "doc", DocID: "d2"},+ },+ },+ }+ prev := nav.GetNavService()+ nav.SetNavService(fake)+ defer func() { nav.SetNavService(prev) }()++ svc := NewDatasetArtifactService()+ n, err := svc.DeleteNavNode(context.Background(), "t1", "kb1", "C1")+ if err != nil {+ t.Fatalf("DeleteNavNode: %v", err)+ }+ if n != 2 {+ t.Errorf("deleted = %d, want 2 (direct doc children d1,d2)", n)+ }+ if len(fake.removedDocs) != 2 {+ t.Errorf("removed docs = %v, want [d1 d2]", fake.removedDocs)+ }+}diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex 38a524fcf0..af9a210a50 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -21,6 +21,7 @@ import ("ragflow/internal/engine""ragflow/internal/engine/types"+ "ragflow/internal/service/nav")// Compile keyword constants used by the knowledge-compilation artifacts stored@@ -601,113 +602,118 @@ func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenantreturn len(ids), nil}-// NavigationItem is a single navigation cluster.+// NavigationItem is a single navigation cluster (REST response shape, kept+// stable for frontend compatibility).type NavigationItem struct {Name string `json:"name"`Title string `json:"title"`Count int `json:"count"`}-// ListNavClusters returns the navigation clusters of a dataset.+// NavChildItem is a single child entry under a navigation cluster (REST+// response shape, kept stable for frontend compatibility).+type NavChildItem struct {+ Name string `json:"name"`+ Title string `json:"title"`+ Count int `json:"count"`+}++// ListNavClusters returns the navigation clusters of a dataset. It is DEPRECATED+// and now delegates to the ES-backed NavService (internal/service datasetnav):+// the previous implementation queried nav_cluster_kwd/count_int fields that+// Python never writes, so it could never read the real nav tree. Do not add+// field-level patches here — route everything through NavService.func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID, datasetID string) ([]NavigationItem, int64, error) {- chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID,- map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}},- []string{"nav_cluster_kwd", "title_kwd", "count_int"}, 0, 10000, nil)+ ns := nav.GetNavService()+ if ns == nil {+ return nil, 0, fmt.Errorf("datasetnav: NavService not initialized (SetNavService must be called at bootstrap)")+ }+ nodes, total, err := ns.ListClusters(ctx, tenantID, datasetID, 0, 10000)if err != nil {return nil, 0, err}- items := make([]NavigationItem, 0, len(chunks))- for _, c := range chunks {- count := intValue(c["count_int"])- items = append(items, NavigationItem{- Name: firstStringValue(c["nav_cluster_kwd"]),- Title: firstStringValue(c["title_kwd"]),- Count: count,- })+ items := make([]NavigationItem, 0, len(nodes))+ for _, n := range nodes {+ items = append(items, NavigationItem{Name: n.Name, Title: n.Description, Count: n.DocCount})}return items, total, nil}-// DeleteNav deletes all navigation clusters of a dataset.-func (s *DatasetArtifactService) DeleteNav(ctx context.Context, tenantID, datasetID string) (int, error) {- docEngine := engine.Get()- if docEngine == nil {- return 0, fmt.Errorf("document engine is not initialized")+// ListNavChildren returns the children of a navigation cluster. DEPRECATED —+// delegates to NavService.ListChildren.+func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, datasetID, name string) ([]NavChildItem, int64, error) {+ ns := nav.GetNavService()+ if ns == nil {+ return nil, 0, fmt.Errorf("datasetnav: NavService not initialized (SetNavService must be called at bootstrap)")}- chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,- map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}}, []string{"id"}, 0, 10000, nil)+ nodes, total, err := ns.ListChildren(ctx, tenantID, datasetID, name, 0, 10000)if err != nil {- return 0, err- }- if len(chunks) == 0 {- return 0, nil- }- ids := make([]string, 0, len(chunks))- for _, c := range chunks {- if id, ok := c["id"].(string); ok {- ids = append(ids, id)- }+ return nil, 0, err}- cond := map[string]interface{}{"id": ids, "kb_id": datasetID}- if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {- return 0, err+ items := make([]NavChildItem, 0, len(nodes))+ for _, n := range nodes {+ items = append(items, NavChildItem{Name: n.Name, Title: n.Description, Count: n.DocCount})}- return len(ids), nil+ return items, total, nil}-// DeleteNavNode deletes a single navigation cluster by name.-func (s *DatasetArtifactService) DeleteNavNode(ctx context.Context, tenantID, datasetID, name string) (int, error) {- docEngine := engine.Get()- if docEngine == nil {- return 0, fmt.Errorf("document engine is not initialized")+// DeleteNav removes the direct nav_doc children of every root cluster of a+// dataset. DEPRECATED — this is the minimal-loop approximation of Python+// delete_nav: it drains only the immediate nav_doc rows under root clusters and+// does NOT implement Python's full subtree traversal or empty-cluster cascade+// cleanup. Prefer the NavService (future work) for a complete delete. Returns+// the number of nav_doc rows removed.+func (s *DatasetArtifactService) DeleteNav(ctx context.Context, tenantID, datasetID string) (int, error) {+ ns := nav.GetNavService()+ if ns == nil {+ return 0, fmt.Errorf("datasetnav: NavService not initialized (SetNavService must be called at bootstrap)")}- chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,- map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}, "nav_cluster_kwd": []string{name}},- []string{"id"}, 0, 10000, nil)+ clusters, _, err := ns.ListClusters(ctx, tenantID, datasetID, 0, 10000)if err != nil {return 0, err}- if len(chunks) == 0 {- return 0, nil- }- ids := make([]string, 0, len(chunks))- for _, c := range chunks {- if id, ok := c["id"].(string); ok {- ids = append(ids, id)+ deleted := 0+ for _, c := range clusters {+ children, _, err := ns.ListChildren(ctx, tenantID, datasetID, c.Name, 0, 10000)+ if err != nil {+ continue+ }+ for _, ch := range children {+ if ch.DocID != "" {+ if err := ns.RemoveDoc(ctx, tenantID, datasetID, ch.DocID); err != nil {+ return deleted, err+ }+ deleted+++ }}}- cond := map[string]interface{}{"id": ids, "kb_id": datasetID}- if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {- return 0, err- }- return len(ids), nil-}--// NavChildItem is a single child entry under a navigation cluster.-type NavChildItem struct {- Name string `json:"name"`- Title string `json:"title"`- Count int `json:"count"`+ return deleted, nil}-// ListNavChildren returns the children of a navigation cluster.-func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, datasetID, name string) ([]NavChildItem, int64, error) {- chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID,- map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}, "nav_cluster_kwd": []string{name}},- []string{"nav_child_kwd", "title_kwd", "count_int"}, 0, 10000, nil)+// DeleteNavNode deletes the direct nav_doc children of a named cluster.+// DEPRECATED — the minimal loop only drains immediate doc children (returns the+// count); it does NOT delete sub-clusters recursively nor perform Python's+// empty-cluster cascade. A full tree-node delete is future NavService work.+func (s *DatasetArtifactService) DeleteNavNode(ctx context.Context, tenantID, datasetID, name string) (int, error) {+ ns := nav.GetNavService()+ if ns == nil {+ return 0, fmt.Errorf("datasetnav: NavService not initialized (SetNavService must be called at bootstrap)")+ }+ // Minimal loop has no per-node delete; drain direct children's docs.+ children, _, err := ns.ListChildren(ctx, tenantID, datasetID, name, 0, 10000)if err != nil {- return nil, 0, err+ return 0, err}- items := make([]NavChildItem, 0, len(chunks))- for _, c := range chunks {- count := intValue(c["count_int"])- items = append(items, NavChildItem{- Name: firstStringValue(c["nav_child_kwd"]),- Title: firstStringValue(c["title_kwd"]),- Count: count,- })+ deleted := 0+ for _, ch := range children {+ if ch.DocID != "" {+ if err := ns.RemoveDoc(ctx, tenantID, datasetID, ch.DocID); err != nil {+ return deleted, err+ }+ deleted+++ }}- return items, total, nil+ return deleted, nil}// SkillTreeItem is a single skill-tree page summary.diff --git a/internal/service/nav/nav.go b/internal/service/nav/nav.gonew file mode 100644index 0000000000..9b590ac2c3--- /dev/null+++ b/internal/service/nav/nav.go@@ -0,0 +1,96 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++// Package nav defines the dataset-navigation service interface. It is kept as a+// dependency-light leaf package so that both the agent tool layer+// (internal/agent/tool) and the concrete implementation+// (internal/service/nlp) can depend on it without creating an import cycle —+// mirroring how RetrievalService lives in the agent tool package. See+// tasks/agentic_search_port_plan.md.+package nav++import (+ "context"+ "sync"+)++// NavNode mirrors Python's _nav_item (dataset_api_service.py).+type NavNode struct {+ Name string // name+ Description string // content_with_weight payload description+ DocCount int // doc_count_int (cluster) or 1 (leaf)+ Type string // "cluster" | "doc"+ DocID string // leaf doc_id; empty for cluster+ HasChildren bool // is_cluster+}++// NavHit is one KNN hit on a nav row.+type NavHit struct {+ Type string // "nav_doc" | "nav_cluster"+ DocID string+ DocIDs []string+ Name string+ Score float64+}++// UpsertDocInput carries the per-document summary to place into the nav tree.+type UpsertDocInput struct {+ TenantID string+ KbID string+ DocID string+ Summary string // document summary text (tree product or page_index summary)+ Embedd []float32 // optional precomputed embedding+}++// NavService is the single read/write entrypoint for a dataset's navigation+// tree. It is the only nav consumer used by agent tools, REST handlers and the+// tree/structure compile-complete hooks.+type NavService interface {+ // UpsertDoc places one document summary into the nav tree (incremental,+ // ES-backed read-modify-write; deterministic placement in the minimal loop).+ UpsertDoc(ctx context.Context, in UpsertDocInput) error+ // RemoveDoc removes a document's nav rows for the given doc. The minimal-loop+ // implementation deletes the nav_doc row(s) for the doc; empty-cluster+ // cascade cleanup is NOT yet implemented.+ RemoveDoc(ctx context.Context, tenantID, kbID, docID string) error++ // Search runs query KNN over nav rows and returns the routed doc ids.+ Search(ctx context.Context, tenantID, kbID, query string, embd []float32, topK int) ([]NavHit, error)+ // ListClusters returns the depth-0 clusters (parent_kwd=root).+ ListClusters(ctx context.Context, tenantID, kbID string, page, pageSize int) ([]NavNode, int64, error)+ // ListChildren returns the direct children of a cluster (parent_kwd=name).+ ListChildren(ctx context.Context, tenantID, kbID, name string, page, pageSize int) ([]NavNode, int64, error)+}++var (+ svcMu sync.RWMutex+ svcInst NavService+)++// SetNavService installs the (production or test) NavService singleton.+func SetNavService(s NavService) {+ svcMu.Lock()+ defer svcMu.Unlock()+ svcInst = s+}++// GetNavService returns the installed NavService. It may return nil until+// SetNavService is called during server bootstrap.+func GetNavService() NavService {+ svcMu.RLock()+ defer svcMu.RUnlock()+ return svcInst+}diff --git a/internal/service/nav_embedder.go b/internal/service/nav_embedder.gonew file mode 100644index 0000000000..6598695291--- /dev/null+++ b/internal/service/nav_embedder.go@@ -0,0 +1,80 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "fmt"+ "strings"+)++// NavEmbedder is the production implementation of nlp.NavEmbedder. It resolves+// the tenant's embedding model on each call and returns float32 vectors (the+// dataset-nav index stores q_<dim>_vec as float). It lives in the service+// package (not nlp) so it can import model_service without an import cycle.+type NavEmbedder struct {+ modelSvc *ModelProviderService+ // embdModelName is the composite embedding model name (e.g.+ // "embedding_model@..." ). Empty falls back to resolving the tenant default.+ embdModelName string+}++// NewNavEmbedder builds the production embedder used by NavService.+func NewNavEmbedder(modelSvc *ModelProviderService, embdModelName string) *NavEmbedder {+ return &NavEmbedder{modelSvc: modelSvc, embdModelName: embdModelName}+}++// Encode embeds texts for the tenant and returns float32 vectors.+func (e *NavEmbedder) Encode(ctx context.Context, tenantID string, texts []string) ([][]float32, error) {+ if e.modelSvc == nil {+ return nil, fmt.Errorf("datasetnav: embedding model service not initialized")+ }+ name := e.embdModelName+ if name == "" {+ name = tenantID // composite name falls back to tenant default resolution+ }+ model, err := e.modelSvc.GetEmbeddingModel(ctx, tenantID, name)+ if err != nil {+ return nil, fmt.Errorf("datasetnav: resolve embedding model for tenant %s: %w", tenantID, err)+ }+ nonEmpty := make([]string, 0, len(texts))+ for _, t := range texts {+ if strings.TrimSpace(t) != "" {+ nonEmpty = append(nonEmpty, t)+ }+ }+ if len(nonEmpty) == 0 {+ return nil, nil+ }+ embeds, err := model.ModelDriver.Embed(ctx, model.ModelName, nonEmpty, model.APIConfig, nil, nil)+ if err != nil {+ return nil, err+ }+ out := make([][]float32, 0, len(embeds))+ for _, e := range embeds {+ out = append(out, toF32(e.Embedding))+ }+ return out, nil+}++func toF32(v []float64) []float32 {+ out := make([]float32, len(v))+ for i, x := range v {+ out[i] = float32(x)+ }+ return out+}diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.gonew file mode 100644index 0000000000..2f8c133f47--- /dev/null+++ b/internal/service/nlp/datasetnav.go@@ -0,0 +1,574 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package nlp++import (+ "context"+ "encoding/json"+ "fmt"+ "strings"++ "ragflow/internal/engine"+ "ragflow/internal/engine/types"+ "ragflow/internal/service/nav"+)++// Dataset-nav constants. These mirror Python's dataset_nav.py: nav rows live in+// the document index as compile_kwd="dataset_nav" rows with available_int=0+// (invisible to the default retriever, which filters available_int=1), and the+// tree is threaded through parent_kwd ("root" for depth-0 clusters).+const (+ navCompileKwd = "dataset_nav"+ navRootParent = "root"++ navMergeThreshold = 0.80 // sim >= this -> merge doc into cluster+ navRecurse = 0.65 // descend while sim >= this+ navMinSim = 0.50 // sim >= this -> new sibling cluster+ navMaxDepth = 6 // max descent levels during best-cluster search+)++// NavEmbedder embeds text. tenantID lets a production implementation resolve+// the tenant's embedding model. Kept as an interface so tests inject a stub.+type NavEmbedder interface {+ Encode(ctx context.Context, tenantID string, texts []string) ([][]float32, error)+}++// NavService is the concrete, ES-backed implementation of nav.NavService.+type NavService struct {+ embed NavEmbedder+ engine engine.DocEngine // optional; falls back to engine.Get() when nil+}++// NewNavService builds the ES-backed NavService. embed may be nil when the+// caller guarantees every UpsertDocInput carries a precomputed Embedd.+func NewNavService(embed NavEmbedder) *NavService {+ return &NavService{embed: embed}+}++func (s *NavService) docEngine() (engine.DocEngine, error) {+ if s.engine != nil {+ return s.engine, nil+ }+ de := engine.Get()+ if de == nil {+ return nil, fmt.Errorf("document engine is not initialized")+ }+ return de, nil+}++// navIndexName returns the tenant document index name (ragflow_<tenantID>).+func (s *NavService) navIndexName(tenantID string) string {+ return fmt.Sprintf("ragflow_%s", tenantID)+}++// navFilter builds the common nav filter that pins compile_kwd.+func navFilter(extra map[string]interface{}) map[string]interface{} {+ f := map[string]interface{}{"compile_kwd": []string{navCompileKwd}}+ for k, v := range extra {+ f[k] = v+ }+ return f+}++// navSearch runs a filtered read over the tenant index for the dataset.+func (s *NavService) navSearch(ctx context.Context, tenantID, kbID string, filter map[string]interface{}, selectFields []string, offset, limit int, matchExprs []interface{}) ([]map[string]interface{}, int64, error) {+ de, err := s.docEngine()+ if err != nil {+ return nil, 0, err+ }+ merged := make(map[string]interface{}, len(filter)+1)+ for k, v := range filter {+ merged[k] = v+ }+ merged["kb_id"] = []string{kbID}+ req := &types.SearchRequest{+ IndexNames: []string{s.navIndexName(tenantID)},+ KbIDs: []string{kbID},+ Offset: offset,+ Limit: limit,+ SelectFields: selectFields,+ Filter: merged,+ MatchExprs: matchExprs,+ }+ res, err := de.Search(ctx, req)+ if err != nil {+ return nil, 0, err+ }+ if res == nil {+ return nil, 0, nil+ }+ return res.Chunks, res.Total, nil+}++// ListClusters returns the depth-0 clusters (parent_kwd=root).+func (s *NavService) ListClusters(ctx context.Context, tenantID, kbID string, page, pageSize int) ([]nav.NavNode, int64, error) {+ if pageSize <= 0 {+ pageSize = 100+ }+ offset := page * pageSize+ chunks, total, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{+ "type_kwd": []string{"nav_cluster"},+ "parent_kwd": []string{navRootParent},+ }),+ []string{"title_kwd", "content_with_weight", "doc_count_int", "type_kwd"}, offset, pageSize, nil)+ if err != nil {+ return nil, 0, err+ }+ nodes := make([]nav.NavNode, 0, len(chunks))+ for _, c := range chunks {+ nodes = append(nodes, s.nodeFromRow(c, "cluster"))+ }+ return nodes, total, nil+}++// ListChildren returns the direct children of a cluster (parent_kwd=name).+func (s *NavService) ListChildren(ctx context.Context, tenantID, kbID, name string, page, pageSize int) ([]nav.NavNode, int64, error) {+ if pageSize <= 0 {+ pageSize = 100+ }+ offset := page * pageSize+ chunks, total, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{"parent_kwd": []string{name}}),+ []string{"title_kwd", "content_with_weight", "doc_count_int", "type_kwd", "doc_id"}, offset, pageSize, nil)+ if err != nil {+ return nil, 0, err+ }+ nodes := make([]nav.NavNode, 0, len(chunks))+ for _, c := range chunks {+ typ := firstStringValue(c["type_kwd"])+ nodeType := "doc"+ if typ == "nav_cluster" {+ nodeType = "cluster"+ }+ nodes = append(nodes, s.nodeFromRow(c, nodeType))+ }+ return nodes, total, nil+}++// nodeFromRow converts an engine row into a NavNode.+func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string) nav.NavNode {+ node := nav.NavNode{+ Name: firstStringValue(row["title_kwd"]),+ DocCount: intValue(row["doc_count_int"]),+ Type: fallbackType,+ DocID: firstStringValue(row["doc_id"]),+ }+ if t := firstStringValue(row["type_kwd"]); t != "" {+ if t == "nav_cluster" {+ node.Type = "cluster"+ } else {+ node.Type = "doc"+ }+ }+ if payload, ok := row["content_with_weight"].(string); ok {+ var m map[string]interface{}+ if err := json.Unmarshal([]byte(payload), &m); err == nil {+ if d, ok := m["description"].(string); ok {+ node.Description = d+ }+ }+ }+ node.HasChildren = node.Type == "cluster"+ if node.DocCount <= 0 {+ node.DocCount = 1+ }+ return node+}++// Search runs query KNN over nav rows and returns routed doc ids.+func (s *NavService) Search(ctx context.Context, tenantID, kbID, query string, embd []float32, topK int) ([]nav.NavHit, error) {+ if topK <= 0 {+ topK = 8+ }+ vec := embd+ if len(vec) == 0 {+ if s.embed == nil {+ return nil, fmt.Errorf("datasetnav: no embedding available for Search")+ }+ embeddings, err := s.embed.Encode(ctx, tenantID, []string{query})+ if err != nil {+ return nil, err+ }+ if len(embeddings) == 0 {+ return nil, fmt.Errorf("datasetnav: embedding produced no vector")+ }+ vec = embeddings[0]+ }+ f64 := f32ToF64Slice(vec)+ chunks, _, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(nil),+ []string{"type_kwd", "title_kwd", "doc_id", "doc_ids_kwd", "_score"}, 0, topK,+ []interface{}{&types.MatchDenseExpr{+ VectorColumnName: fmt.Sprintf("q_%d_vec", len(f64)),+ EmbeddingData: f64,+ EmbeddingDataType: "float",+ DistanceType: "cosine",+ TopN: topK,+ ExtraOptions: map[string]interface{}{"similarity": 0.0},+ }})+ if err != nil {+ return nil, err+ }+ hits := make([]nav.NavHit, 0, len(chunks))+ for _, c := range chunks {+ h := nav.NavHit{+ Type: firstStringValue(c["type_kwd"]),+ Name: firstStringValue(c["title_kwd"]),+ DocID: firstStringValue(c["doc_id"]),+ }+ if sc, ok := c["_score"].(float64); ok {+ h.Score = sc+ } else if sc, ok := c["_score"].(float32); ok {+ h.Score = float64(sc)+ }+ if ds, ok := c["doc_ids_kwd"].([]interface{}); ok {+ for _, d := range ds {+ if dd, ok := d.(string); ok {+ h.DocIDs = append(h.DocIDs, dd)+ }+ }+ }+ hits = append(hits, h)+ }+ return hits, nil+}++// UpsertDoc places one document summary into the nav tree. Minimal closed loop:+// deterministic placement (KNN find best cluster -> merge if sim>=0.80, else a+// new root-level cluster). No LLM, no split/rebalance, no cascade cleanup.+func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error {+ if strings.TrimSpace(in.Summary) == "" {+ return nil+ }+ de, err := s.docEngine()+ if err != nil {+ return err+ }+ if s.embed == nil && len(in.Embedd) == 0 {+ return fmt.Errorf("datasetnav: embedder required for UpsertDoc")+ }+ vec := in.Embedd+ if len(vec) == 0 {+ embeddings, err := s.embed.Encode(ctx, in.TenantID, []string{in.Summary})+ if err != nil {+ return err+ }+ if len(embeddings) == 0 {+ return nil+ }+ vec = embeddings[0]+ }++ // storeGet: skip if a nav_doc for this doc already exists with same summary.+ existing, _, err := s.navSearch(ctx, in.TenantID, in.KbID,+ navFilter(map[string]interface{}{"doc_id": []string{in.DocID}}),+ []string{"content_with_weight"}, 0, 1, nil)+ if err != nil {+ return err+ }+ if len(existing) > 0 {+ if payload, ok := existing[0]["content_with_weight"].(string); ok {+ var m map[string]interface{}+ if err := json.Unmarshal([]byte(payload), &m); err == nil {+ if d, _ := m["description"].(string); d == in.Summary {+ return nil // unchanged+ }+ }+ }+ // Changed summary: remove the old nav_doc first (no cascade in minimal loop).+ if _, err := s.deleteNavDoc(ctx, in.TenantID, in.KbID, in.DocID); err != nil {+ return err+ }+ }++ bestName, sim, bestDepth, err := s.findBestCluster(ctx, in.TenantID, in.KbID, vec)+ if err != nil {+ return err+ }+ idx := s.navIndexName(in.TenantID)++ if bestName != "" && sim >= navMergeThreshold {+ parent := bestName+ if err := s.appendDocToCluster(ctx, de, in.TenantID, in.KbID, bestName, in.DocID); err != nil {+ return err+ }+ _, err = de.InsertChunks(ctx, []map[string]interface{}{{+ "compile_kwd": navCompileKwd,+ "available_int": 0,+ "type_kwd": "nav_doc",+ "title_kwd": in.DocID,+ "parent_kwd": parent,+ // The nav_doc sits one level below its (possibly nested) parent+ // cluster, so its depth is parentDepth+1 — not a hard-coded 1.+ "depth_int": bestDepth + 1,+ "doc_id": in.DocID,+ "doc_count_int": 1,+ "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_doc", "description": in.Summary}),+ "q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),+ }}, idx, in.KbID)+ return err+ }++ // A similar-but-not-mergeable cluster creates a sibling sub-cluster (Python+ // _MIN_SIM=0.50); otherwise a fresh root cluster. This keeps the nav tree+ // from degrading into one root per document.+ parent := navRootParent+ depth := 0+ if bestName != "" && sim >= navMinSim {+ parent = bestName+ // A sibling of the (possibly nested) best cluster is one level deeper+ // than it, so depth = parentDepth+1 — not a hard-coded 1.+ depth = bestDepth + 1+ }+ name := navDocName(in.DocID, in.Summary)+ _, err = de.InsertChunks(ctx, []map[string]interface{}{{+ "compile_kwd": navCompileKwd,+ "available_int": 0,+ "type_kwd": "nav_cluster",+ "title_kwd": name,+ "parent_kwd": parent,+ "depth_int": depth,+ "doc_count_int": 1,+ "doc_ids_kwd": []string{in.DocID},+ "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_cluster", "description": in.Summary}),+ "q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),+ }}, idx, in.KbID)+ return err+}++// findBestCluster finds the best-matching cluster via level-by-level descent+// (mirroring Python _find_best_cluster). It KNNs the current level's clusters+// and, when the best match is >= recurse threshold, descends into that cluster's+// children. Returns the best cluster name, similarity, and its depth (0 = root)+// so callers can assign consistent child depth_int values.+func (s *NavService) findBestCluster(ctx context.Context, tenantID, kbID string, vec []float32) (string, float64, int, error) {+ f64 := f32ToF64Slice(vec)+ parent := navRootParent+ bestName := ""+ bestSim := 0.0+ bestDepth := 0+ for level := 0; level < navMaxDepth; level++ {+ // KNN among clusters whose parent is the current level.+ chunks, _, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{+ "type_kwd": []string{"nav_cluster"},+ "parent_kwd": []string{parent},+ }),+ []string{"title_kwd", "_score"}, 0, 1,+ []interface{}{&types.MatchDenseExpr{+ VectorColumnName: fmt.Sprintf("q_%d_vec", len(f64)),+ EmbeddingData: f64,+ EmbeddingDataType: "float",+ DistanceType: "cosine",+ TopN: 1,+ ExtraOptions: map[string]interface{}{"similarity": 0.0},+ }})+ if err != nil {+ return "", 0, 0, err+ }+ if len(chunks) == 0 {+ break+ }+ name := firstStringValue(chunks[0]["title_kwd"])+ sim := rowScore(chunks[0])+ // Keep the STRONGEST match seen so far across all levels, so a strong+ // ancestor is never displaced by a weaker descendant. Record its depth+ // so the caller can set consistent child depth_int values.+ if sim > bestSim {+ bestName, bestSim, bestDepth = name, sim, level+ }+ // Descend only while the current match is strong enough that a deeper+ // child could be a better target.+ if sim < navRecurse {+ break+ }+ parent = name+ }+ return bestName, bestSim, bestDepth, nil+}++// rowScore extracts the engine's _score field as float64.+func rowScore(row map[string]interface{}) float64 {+ if sc, ok := row["_score"].(float64); ok {+ return sc+ }+ if sc, ok := row["_score"].(float32); ok {+ return float64(sc)+ }+ return 0+}++// appendDocToCluster appends a doc id to a cluster's doc_ids_kwd and bumps its+// doc_count_int. Implemented as a read-modify-write.+func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docID string) error {+ chunks, _, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),+ []string{"doc_ids_kwd", "doc_count_int"}, 0, 1, nil)+ if err != nil {+ return err+ }+ if len(chunks) == 0 {+ return nil+ }+ ids := []string{}+ if raw, ok := chunks[0]["doc_ids_kwd"].([]interface{}); ok {+ for _, d := range raw {+ if dd, ok := d.(string); ok {+ ids = append(ids, dd)+ }+ }+ }+ found := false+ for _, id := range ids {+ if id == docID {+ found = true+ break+ }+ }+ if !found {+ ids = append(ids, docID)+ }+ count := intValue(chunks[0]["doc_count_int"])+ if !found {+ count+++ }+ // Pin the update to the nav_cluster row only: a regular chunk sharing the+ // same title_kwd must never be clobbered. The read-modify-write here is+ // expected to run under a per-dataset lock held by the UpsertDoc caller;+ // without it, concurrent appends to the same cluster can lose updates.+ return de.UpdateChunks(ctx,+ map[string]interface{}{+ "compile_kwd": []string{navCompileKwd},+ "type_kwd": []string{"nav_cluster"},+ "title_kwd": []string{clusterName},+ "kb_id": kbID,+ },+ map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count},+ s.navIndexName(tenantID), kbID)+}++// deleteNavDoc deletes a nav_doc row by doc_id.+func (s *NavService) deleteNavDoc(ctx context.Context, tenantID, kbID, docID string) (int64, error) {+ de, err := s.docEngine()+ if err != nil {+ return 0, err+ }+ chunks, _, err := s.navSearch(ctx, tenantID, kbID,+ navFilter(map[string]interface{}{"doc_id": []string{docID}}),+ []string{"id"}, 0, 100, nil)+ if err != nil {+ return 0, err+ }+ ids := make([]string, 0, len(chunks))+ for _, c := range chunks {+ if id, ok := c["id"].(string); ok {+ ids = append(ids, id)+ }+ }+ if len(ids) == 0 {+ return 0, nil+ }+ return de.DeleteChunks(ctx, map[string]interface{}{"id": ids, "kb_id": kbID}, s.navIndexName(tenantID), kbID)+}++// RemoveDoc removes a document's nav_doc (no cascade cleanup in minimal loop).+func (s *NavService) RemoveDoc(ctx context.Context, tenantID, kbID, docID string) error {+ _, err := s.deleteNavDoc(ctx, tenantID, kbID, docID)+ return err+}++// f32ToF64Slice converts a float32 vector to float64.+func f32ToF64Slice(v []float32) []float64 {+ out := make([]float64, len(v))+ for i, x := range v {+ out[i] = float64(x)+ }+ return out+}++// payloadJSONNav marshals a nav payload map into the content_with_weight JSON.+func payloadJSONNav(v map[string]interface{}) string {+ b, err := json.Marshal(v)+ if err != nil {+ return "{}"+ }+ return string(b)+}++// navDocName builds a deterministic cluster name for the minimal loop.+func navDocName(docID, summary string) string {+ return docID + "_" + contentHash8(summary)+}++// contentHash8 is a stable 8-char hash.+func contentHash8(s string) string {+ h := uint32(2166136261)+ for i := 0; i < len(s); i++ {+ h ^= uint32(s[i])+ h *= 16777619+ }+ return fmt.Sprintf("%08x", h)+}++// firstStringValue returns the first string value of a (possibly list-wrapped)+// engine field.+func firstStringValue(v interface{}) string {+ switch tv := v.(type) {+ case string:+ return tv+ case []string:+ if len(tv) > 0 {+ return tv[0]+ }+ case []interface{}:+ if len(tv) > 0 {+ if s, ok := tv[0].(string); ok {+ return s+ }+ }+ }+ return ""+}++// intValue returns the integer value of an engine field.+func intValue(v interface{}) int {+ switch tv := v.(type) {+ case float64:+ return int(tv)+ case float32:+ return int(tv)+ case int:+ return tv+ case int64:+ return int(tv)+ case []float64:+ if len(tv) > 0 {+ return int(tv[0])+ }+ case []interface{}:+ if len(tv) > 0 {+ switch n := tv[0].(type) {+ case float64:+ return int(n)+ case int:+ return n+ }+ }+ }+ return 0+}diff --git a/internal/service/nlp/datasetnav_integration_test.go b/internal/service/nlp/datasetnav_integration_test.gonew file mode 100644index 0000000000..194bf36335--- /dev/null+++ b/internal/service/nlp/datasetnav_integration_test.go@@ -0,0 +1,139 @@+//go:build integration+// +build integration++package nlp++import (+ "context"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "ragflow/internal/engine"+ "ragflow/internal/engine/types"+ "ragflow/internal/server"+ "ragflow/internal/service/nav"++ "go.uber.org/zap"+)++// repoRootOf walks up from the package directory to the repository root (the+// dir containing go.mod). Kept local to this test package so it needs no shared+// helper from another package.+func repoRootOf(t *testing.T) string {+ t.Helper()+ dir, err := os.Getwd()+ if err != nil {+ t.Fatalf("getwd: %v", err)+ }+ for {+ if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {+ return dir+ }+ parent := filepath.Dir(dir)+ if parent == dir {+ t.Fatal("repository root (go.mod) not found above cwd")+ }+ dir = parent+ }+}++// findNavRow reads the nav row for a doc directly from the document engine.+func findNavRow(t *testing.T, tenantID, kbID, docID string) map[string]interface{} {+ t.Helper()+ de := engine.Get()+ if de == nil {+ t.Skip("no live document engine")+ }+ idx := "ragflow_" + tenantID+ req := &types.SearchRequest{+ IndexNames: []string{idx},+ Filter: map[string]interface{}{"doc_id": []string{docID}, "compile_kwd": []string{"dataset_nav"}},+ SelectFields: []string{"available_int", "compile_kwd", "type_kwd"},+ Limit: 10,+ }+ res, err := de.Search(context.Background(), req)+ if err != nil {+ t.Fatalf("search nav row: %v", err)+ }+ for _, row := range res.Chunks {+ return row+ }+ t.Fatalf("no nav row found for doc %s", docID)+ return nil+}++// TestDatasetNav_AvailableIntZero_Isolation is an integration test against a real+// ES/Infinity backend (requires conf/service_conf.yaml + a live document store).+// It verifies acceptance criterion #6: a nav row written with available_int=0 is+// invisible to the default retriever (which filters available_int=1) but IS+// reachable through NavService.Search.+//+// Run with: bash build.sh --test-integration ./internal/service/nlp/...+func TestDatasetNav_AvailableIntZero_Isolation(t *testing.T) {+ server.SetLogger(zap.NewNop())+ configPath := filepath.Join(repoRootOf(t), "conf", "service_conf.yaml")+ if err := server.Init(configPath); err != nil {+ t.Fatalf("init service config: %v", err)+ }+ if err := engine.Init(); err != nil {+ t.Fatalf("init document engine: %v", err)+ }+ if engine.Get() == nil {+ t.Skip("no live document engine configured")+ }++ tenantID := "navint_t1"+ kbID := "navint_kb1"+ docID := "navint_doc1"++ ns := NewNavService(stubNavEmbedder{})+ if err := ns.UpsertDoc(context.Background(), nav.UpsertDocInput{+ TenantID: tenantID, KbID: kbID, DocID: docID, Summary: "rocket propulsion integration evidence",+ }); err != nil {+ t.Fatalf("upsert nav doc: %v", err)+ }+ t.Cleanup(func() { _ = ns.RemoveDoc(context.Background(), tenantID, kbID, docID) })++ // NavService.Search must find the nav row (reads nav rows directly).+ hits, err := ns.Search(context.Background(), tenantID, kbID, "rocket propulsion", nil, 5)+ if err != nil {+ t.Fatalf("nav search: %v", err)+ }+ if len(hits) == 0 {+ t.Fatal("NavService.Search returned no hits; nav row is not reachable")+ }++ // The written nav row must carry compile_kwd=dataset_nav and available_int=0,+ // so the default retriever (available_int=1 filter) will not surface it.+ row := findNavRow(t, tenantID, kbID, docID)+ // compile_kwd may come back list-wrapped by the engine, so use firstStrOrSlice.+ if ck := firstStrOrSlice(row["compile_kwd"]); !strings.Contains(ck, "dataset_nav") {+ t.Errorf("nav row compile_kwd = %q, want dataset_nav", ck)+ }+ avail := intValue(row["available_int"])+ if avail != 0 {+ t.Errorf("nav row available_int = %d, want 0 (so it is hidden from the default retriever)", avail)+ }+}++// firstStrOrSlice returns the first string of a value that may be a plain string+// or a list-wrapped string (engine fields are often returned as []interface{}).+func firstStrOrSlice(v interface{}) string {+ switch tv := v.(type) {+ case string:+ return tv+ case []string:+ if len(tv) > 0 {+ return tv[0]+ }+ case []interface{}:+ if len(tv) > 0 {+ if s, ok := tv[0].(string); ok {+ return s+ }+ }+ }+ return ""+}diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.gonew file mode 100644index 0000000000..bc4c52a548--- /dev/null+++ b/internal/service/nlp/datasetnav_test.go@@ -0,0 +1,390 @@+package nlp++import (+ "context"+ "math"+ "testing"++ "ragflow/internal/engine/types"+ "ragflow/internal/service/nav"++ "gorm.io/gorm"+)++// memNavEngine is an in-memory DocEngine double sufficient for the datasetnav+// minimal closed loop. It stores rows in a slice and supports the filtered+// search / insert / update / delete operations NavService uses. Dense-vector+// KNN is approximated by a deterministic cosine over a synthetic q_<dim>_vec.+type memNavEngine struct {+ rows []map[string]interface{}+ nextID int+}++func newMemNavEngine() *memNavEngine { return &memNavEngine{} }++func (m *memNavEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _ string, datasetID string) ([]string, error) {+ ids := make([]string, 0, len(chunks))+ for _, c := range chunks {+ m.nextID+++ id := "nav" + strconvItoa(m.nextID)+ cp := make(map[string]interface{}, len(c)+2)+ for k, v := range c {+ cp[k] = v+ }+ cp["id"] = id+ cp["kb_id"] = datasetID+ m.rows = append(m.rows, cp)+ ids = append(ids, id)+ }+ return ids, nil+}++func (m *memNavEngine) UpdateChunks(_ context.Context, cond map[string]interface{}, newValue map[string]interface{}, _ string, _ string) error {+ for _, r := range m.rows {+ if matchNavRow(r, cond) {+ for k, v := range newValue {+ r[k] = v+ }+ }+ }+ return nil+}++func (m *memNavEngine) DeleteChunks(_ context.Context, cond map[string]interface{}, _ string, _ string) (int64, error) {+ out := m.rows[:0]+ var deleted int64+ for _, r := range m.rows {+ if matchNavRow(r, cond) {+ deleted+++ continue+ }+ out = append(out, r)+ }+ m.rows = out+ return deleted, nil+}++func (m *memNavEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ var matched []map[string]interface{}+ hasDense := len(req.MatchExprs) > 0+ var queryVec []float64+ if hasDense {+ if de, ok := req.MatchExprs[0].(*types.MatchDenseExpr); ok {+ queryVec = de.EmbeddingData+ }+ }+ for _, r := range m.rows {+ if !matchNavRow(r, req.Filter) {+ continue+ }+ if hasDense {+ col := "q_" + strconvItoa(len(queryVec)) + "_vec"+ rv, ok := r[col].([]float64)+ if !ok {+ continue+ }+ r["_score"] = cosineNav(rv, queryVec)+ }+ matched = append(matched, r)+ }+ if hasDense {+ for i := 1; i < len(matched); i++ {+ for j := i; j > 0 && scoreNavOf(matched[j]) > scoreNavOf(matched[j-1]); j-- {+ matched[j], matched[j-1] = matched[j-1], matched[j]+ }+ }+ }+ offset, limit := req.Offset, req.Limit+ if offset > len(matched) {+ offset = len(matched)+ }+ end := offset + limit+ if limit <= 0 || end > len(matched) {+ end = len(matched)+ }+ return &types.SearchResult{Chunks: matched[offset:end], Total: int64(len(matched))}, nil+}++func (m *memNavEngine) DropChunkStore(context.Context, string, string) error { return nil }+func (m *memNavEngine) ChunkStoreExists(context.Context, string, string) (bool, error) {+ return true, nil+}+func (m *memNavEngine) Close() error { return nil }+func (m *memNavEngine) Ping(context.Context) error { return nil }+func (m *memNavEngine) GetType() string { return "mem" }+func (m *memNavEngine) SupportsPageRank() bool { return false }+func (m *memNavEngine) CreateChunkStore(context.Context, string, string, int, string) error {+ return nil+}+func (m *memNavEngine) GetChunk(context.Context, string, string, []string) (interface{}, error) {+ return nil, nil+}+func (m *memNavEngine) CreateMetadataStore(context.Context, string) error { return nil }+func (m *memNavEngine) InsertMetadata(context.Context, []map[string]interface{}, string) ([]string, error) {+ return nil, nil+}+func (m *memNavEngine) UpdateMetadata(context.Context, string, string, map[string]interface{}, string) error {+ return nil+}+func (m *memNavEngine) DeleteMetadata(context.Context, map[string]interface{}, string) (int64, error) {+ return 0, nil+}+func (m *memNavEngine) DeleteMetadataKeys(context.Context, string, string, []string, string) error {+ return nil+}+func (m *memNavEngine) DropMetadataStore(context.Context, string) error { return nil }+func (m *memNavEngine) MetadataStoreExists(context.Context, string) (bool, error) {+ return true, nil+}+func (m *memNavEngine) SearchMetadata(context.Context, *types.SearchMetadataRequest) (*types.SearchMetadataResult, error) {+ return nil, nil+}+func (m *memNavEngine) IndexDocument(context.Context, string, string, interface{}) error {+ return nil+}+func (m *memNavEngine) DeleteDocument(context.Context, string, string) error { return nil }+func (m *memNavEngine) BulkIndex(context.Context, string, []interface{}) (interface{}, error) {+ return nil, nil+}+func (m *memNavEngine) GetFields([]map[string]interface{}, []string) map[string]map[string]interface{} {+ return nil+}+func (m *memNavEngine) GetAggregation([]map[string]interface{}, string) []map[string]interface{} {+ return nil+}+func (m *memNavEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {+ return nil+}+func (m *memNavEngine) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {+ return nil, nil+}+func (m *memNavEngine) GetChunkIDs([]map[string]interface{}) []string { return nil }+func (m *memNavEngine) KNNScores(context.Context, []map[string]interface{}, []float64, int) (map[string]interface{}, error) {+ return nil, nil+}+func (m *memNavEngine) GetScores(map[string]interface{}) map[string]float64 { return nil }+func (m *memNavEngine) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []string, []map[string]interface{}, string) []string {+ return nil+}++func matchNavRow(row map[string]interface{}, cond map[string]interface{}) bool {+ for k, v := range cond {+ rv, ok := row[k]+ if !ok {+ return false+ }+ switch want := v.(type) {+ case []string:+ ok = false+ for _, w := range want {+ if rv == w {+ ok = true+ break+ }+ }+ if !ok {+ return false+ }+ default:+ if rv != v {+ return false+ }+ }+ }+ return true+}++func scoreNavOf(r map[string]interface{}) float64 {+ switch s := r["_score"].(type) {+ case float64:+ return s+ case float32:+ return float64(s)+ }+ return 0+}++func cosineNav(a, b []float64) float64 {+ if len(a) == 0 || len(b) == 0 || len(a) != len(b) {+ return 0+ }+ var dot, na, nb float64+ for i := range a {+ dot += a[i] * b[i]+ na += a[i] * a[i]+ nb += b[i] * b[i]+ }+ if na == 0 || nb == 0 {+ return 0+ }+ return dot / (math.Sqrt(na) * math.Sqrt(nb))+}++func strconvItoa(n int) string {+ if n == 0 {+ return "0"+ }+ digits := []byte{}+ for n > 0 {+ digits = append([]byte{byte('0' + n%10)}, digits...)+ n /= 10+ }+ return string(digits)+}++// stubNavEmbedder returns a fixed deterministic vector per distinct text.+type stubNavEmbedder struct{}++func (stubNavEmbedder) Encode(_ context.Context, _ string, texts []string) ([][]float32, error) {+ out := make([][]float32, len(texts))+ for i, t := range texts {+ dim := 8+ v := make([]float32, dim)+ for d := 0; d < dim; d++ {+ v[d] = float32(int(t[0]) + d)+ }+ out[i] = v+ }+ return out, nil+}++func newTestNav(eng *memNavEngine) *NavService {+ ns := NewNavService(stubNavEmbedder{})+ ns.engine = eng+ return ns+}++// TestNavService_UpsertDoc_WritesNavRow asserts acceptance #1: after UpsertDoc+// the row carries both compile_kwd=dataset_nav and available_int=0.+func TestNavService_UpsertDoc_WritesNavRow(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ if err := ns.UpsertDoc(context.Background(), navUpsertInput("t1", "kb1", "d1", "alpha")); err != nil {+ t.Fatalf("UpsertDoc: %v", err)+ }+ if len(eng.rows) == 0 {+ t.Fatal("expected at least one nav row")+ }+ row := eng.rows[0]+ if row["compile_kwd"] != "dataset_nav" {+ t.Errorf("compile_kwd = %v, want dataset_nav", row["compile_kwd"])+ }+ if row["available_int"] != 0 {+ t.Errorf("available_int = %v, want 0", row["available_int"])+ }+ if row["type_kwd"] != "nav_cluster" {+ t.Errorf("type_kwd = %v, want nav_cluster", row["type_kwd"])+ }+ if row["parent_kwd"] != "root" {+ t.Errorf("parent_kwd = %v, want root", row["parent_kwd"])+ }+}++// TestNavService_ListClusters_FiltersRoot asserts acceptance #3.+func TestNavService_ListClusters_FiltersRoot(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ if err := ns.UpsertDoc(context.Background(), navUpsertInput("t1", "kb1", "d1", "aaa")); err != nil {+ t.Fatal(err)+ }+ clusters, total, err := ns.ListClusters(context.Background(), "t1", "kb1", 0, 10)+ if err != nil {+ t.Fatal(err)+ }+ if total != 1 || len(clusters) != 1 {+ t.Fatalf("expected 1 root cluster, got total=%d len=%d", total, len(clusters))+ }+ if clusters[0].Type != "cluster" {+ t.Errorf("cluster type = %s, want cluster", clusters[0].Type)+ }+ if clusters[0].DocCount < 1 {+ t.Errorf("cluster doc_count = %d, want >=1", clusters[0].DocCount)+ }+}++// TestNavService_Search_ReturnsHit asserts acceptance #5.+func TestNavService_Search_ReturnsHit(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ if err := ns.UpsertDoc(context.Background(), navUpsertInput("t1", "kb1", "d1", "aaa")); err != nil {+ t.Fatal(err)+ }+ hits, err := ns.Search(context.Background(), "t1", "kb1", "aaa", nil, 5)+ if err != nil {+ t.Fatal(err)+ }+ if len(hits) == 0 {+ t.Fatal("expected hits")+ }+ if hits[0].Name == "" {+ t.Error("hit name empty")+ }+}++// TestNavService_Acceptance4_ListChildren asserts acceptance #4: ListChildren+// returns only the rows whose parent_kwd=name. Two docs with identical stub+// vectors merge into one root cluster: the first becomes the cluster itself,+// the second merges in as a nav_doc (parent_kwd=clusterName). So the cluster+// doc_count reflects both docs, and exactly one nav_doc sits under it.+func TestNavService_Acceptance4_ListChildren(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ // Two docs that merge into one root cluster (same first char -> identical+ // stub vectors -> sim=1.0 >= merge threshold).+ if err := ns.UpsertDoc(context.Background(), navUpsertInput("t1", "kb1", "d1", "aaa one")); err != nil {+ t.Fatal(err)+ }+ if err := ns.UpsertDoc(context.Background(), navUpsertInput("t1", "kb1", "d2", "aaa two")); err != nil {+ t.Fatal(err)+ }+ clusters, _, err := ns.ListClusters(context.Background(), "t1", "kb1", 0, 10)+ if err != nil {+ t.Fatal(err)+ }+ if len(clusters) != 1 {+ t.Fatalf("expected 1 cluster, got %d", len(clusters))+ }+ if clusters[0].DocCount != 2 {+ t.Fatalf("cluster doc_count = %d, want 2 (both docs merged into the cluster)", clusters[0].DocCount)+ }+ name := clusters[0].Name+ children, total, err := ns.ListChildren(context.Background(), "t1", "kb1", name, 0, 10)+ if err != nil {+ t.Fatal(err)+ }+ // Exactly one nav_doc (for d2) sits under the cluster; d1 is the cluster.+ if total != 1 || len(children) != 1 {+ t.Fatalf("expected 1 child under cluster, got total=%d len=%d", total, len(children))+ }+ if children[0].DocID != "d2" {+ t.Errorf("child doc_id = %q, want d2", children[0].DocID)+ }+ if children[0].Type != "doc" {+ t.Errorf("child type = %q, want doc", children[0].Type)+ }+}++// TestNavService_NavDocDepth asserts a nav_doc merged under a root cluster+// (depth 0) gets depth_int = parentDepth+1 = 1, not a hard-coded value.+func TestNavService_NavDocDepth(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ if err := ns.UpsertDoc(context.Background(), navUpsertInput("t1", "kb1", "d1", "aaa one")); err != nil {+ t.Fatal(err)+ }+ if err := ns.UpsertDoc(context.Background(), navUpsertInput("t1", "kb1", "d2", "aaa two")); err != nil {+ t.Fatal(err)+ }+ // The nav_doc for d2 sits under the root cluster; its depth_int must be 1.+ for _, row := range eng.rows {+ if row["doc_id"] == "d2" {+ if d, ok := row["depth_int"].(int); !ok || d != 1 {+ t.Errorf("nav_doc d2 depth_int = %v, want 1 (parentDepth 0 + 1)", row["depth_int"])+ }+ }+ }+}++func navUpsertInput(tenant, kb, doc, summary string) nav.UpsertDocInput {+ return nav.UpsertDocInput{TenantID: tenant, KbID: kb, DocID: doc, Summary: summary}+}