RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Port agentic RAG PRs to GO (#19527)
qinling0210ce34ec42f5e7
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex 314a07da3d..bdee103835 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -650,8 +650,9 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg}defer tokenizer.Close()- // Fail fast if the cl100k_base BPE table is missing: without it- // NumTokensFromString silently returns 0, corrupting every token budget.+ // Fail fast if the cl100k_base BPE table is missing. NumTokensFromString /+ // TrimContentToTokenLimit now panic rather than degrading silently, so this+ // trades a mid-request panic for a clear startup failure.if err := tokenizer.InitCL100KEncoder(); err != nil {common.Fatal("Failed to initialize cl100k_base tokenizer", zap.Error(err))}@@ -800,8 +801,9 @@ func runAPI(ctx context.Context, args *serverArgs) error {}defer tokenizer.Close()- // Fail fast if the cl100k_base BPE table is missing: without it- // NumTokensFromString silently returns 0, corrupting every token budget.+ // Fail fast if the cl100k_base BPE table is missing. NumTokensFromString /+ // TrimContentToTokenLimit now panic rather than degrading silently, so this+ // trades a mid-request panic for a clear startup failure.if err := tokenizer.InitCL100KEncoder(); err != nil {common.Fatal("Failed to initialize cl100k_base tokenizer", zap.Error(err))}diff --git a/internal/agent/chat/message_fit.go b/internal/agent/chat/message_fit.goindex 24c132340a..b7dcbf21cd 100644--- a/internal/agent/chat/message_fit.go+++ b/internal/agent/chat/message_fit.go@@ -51,8 +51,10 @@ func ContextFitBudget(maxLength int) int {}// validateFittedMessages checks that the fitted message list is non-empty-// and the last message is a non-empty user turn (content or multi-modal-// parts). Returns an error string on failure, empty string on success.+// and the last message is a user turn that still carries something usable —+// non-empty text, or a real media payload. Returns an error string on failure,+// empty string on success.+//// Python requires len >= 2 because the system prompt is always injected// upstream; Go allows len >= 1 because the system message may be embedded// inside msgs (from buildMessagesWithImages) or absent entirely.@@ -64,36 +66,70 @@ func validateFittedMessages(msgFit []schema.Message) string {if last.Role != schema.User {return "**ERROR**: LLM last message is not a user turn after prompt fitting; check model content_length context setting"}- if strings.TrimSpace(last.Content) == "" && len(last.UserInputMultiContent) == 0 {+ if strings.TrimSpace(last.Content) == "" && !hasUsableContent(last.UserInputMultiContent) {return "**ERROR**: LLM user message is empty after prompt fitting; check model content_length context setting"}return ""}+// hasUsableContent reports whether the multi-modal parts still give the model+// something to work with: non-empty text, or a media payload with a url or+// inline data. Parts merely EXISTING is not enough — a text-only turn whose+// text the fitting trimmed away leaves a non-empty slice holding an empty text+// part, and sending that turn would ask the model to answer nothing (Python's+// validate_fitted_messages rejects an empty user content the same way).+func hasUsableContent(parts []schema.MessageInputPart) bool {+ for _, p := range parts {+ if p.Type == schema.ChatMessagePartTypeText {+ if strings.TrimSpace(p.Text) != "" {+ return true+ }+ continue+ }+ for _, c := range payloadCommons(p) {+ hasURL := c.URL != nil && strings.TrimSpace(*c.URL) != ""+ hasData := c.Base64Data != nil && strings.TrimSpace(*c.Base64Data) != ""+ if hasURL || hasData {+ return true+ }+ }+ }+ return false+}++// payloadCommons returns the media payloads a part carries, if any.+func payloadCommons(p schema.MessageInputPart) []*schema.MessagePartCommon {+ out := make([]*schema.MessagePartCommon, 0, 4)+ if p.Image != nil {+ out = append(out, &p.Image.MessagePartCommon)+ }+ if p.Audio != nil {+ out = append(out, &p.Audio.MessagePartCommon)+ }+ if p.Video != nil {+ out = append(out, &p.Video.MessagePartCommon)+ }+ if p.File != nil {+ out = append(out, &p.File.MessagePartCommon)+ }+ return out+}+// FitMessages applies message_fit_in semantics to the given messages and// validates that the result ends with a non-empty user turn. Returns the// fitted messages and an error string (empty on success).// Mirrors Python's LLM.fit_messages in PR #16413.func FitMessages(systemPrompt string, msgs []schema.Message, maxLength int) ([]schema.Message, string) {// Deep-copy msgs (mirrors Python's deepcopy) to avoid mutating caller's slice.+ // Copy the WHOLE message first: the fitted history must keep every field the+ // model layer relies on (tool calls, tool ids, reasoning, extras, ...), which+ // a Role/Content/parts literal would silently drop. Then deep-copy only the+ // reference-backed fields fitting rewrites.copied := make([]schema.Message, len(msgs))for i, m := range msgs {- cloned := slices.Clone(m.UserInputMultiContent)- for j, p := range cloned {- if p.Image != nil {- imgCopy := *p.Image- if p.Image.URL != nil {- u := *p.Image.URL- imgCopy.URL = &u- }- cloned[j].Image = &imgCopy- }- }- copied[i] = schema.Message{- Role: m.Role,- Content: m.Content,- UserInputMultiContent: cloned,- }+ cloned := m+ cloned.UserInputMultiContent = deepCopyInputParts(m.UserInputMultiContent)+ copied[i] = cloned}// Convert to messagefit.Message. Track where each entry's text lives@@ -176,3 +212,47 @@ func FitMessages(systemPrompt string, msgs []schema.Message, maxLength int) ([]s}return result, validateFittedMessages(result)}++// deepCopyInputParts clones the multi-modal parts — the slice, the media+// payload structs and their url/base64 pointers — so fitting can rewrite a+// part's text or drop parts without touching the caller's message.+func deepCopyInputParts(parts []schema.MessageInputPart) []schema.MessageInputPart {+ out := slices.Clone(parts)+ for j, p := range out {+ if p.Image != nil {+ img := *p.Image+ img.MessagePartCommon = clonePartCommon(img.MessagePartCommon)+ out[j].Image = &img+ }+ if p.Audio != nil {+ audio := *p.Audio+ audio.MessagePartCommon = clonePartCommon(audio.MessagePartCommon)+ out[j].Audio = &audio+ }+ if p.Video != nil {+ video := *p.Video+ video.MessagePartCommon = clonePartCommon(video.MessagePartCommon)+ out[j].Video = &video+ }+ if p.File != nil {+ file := *p.File+ file.MessagePartCommon = clonePartCommon(file.MessagePartCommon)+ out[j].File = &file+ }+ }+ return out+}++// clonePartCommon copies a payload's pointer-backed fields so the fitted message+// never aliases the caller's payload.+func clonePartCommon(c schema.MessagePartCommon) schema.MessagePartCommon {+ if c.URL != nil {+ u := *c.URL+ c.URL = &u+ }+ if c.Base64Data != nil {+ d := *c.Base64Data+ c.Base64Data = &d+ }+ return c+}diff --git a/internal/agent/chat/message_fit_test.go b/internal/agent/chat/message_fit_test.goindex 3820fedb14..6374368db9 100644--- a/internal/agent/chat/message_fit_test.go+++ b/internal/agent/chat/message_fit_test.go@@ -209,3 +209,111 @@ func TestFitMessages_ImageOnlyLastTurnOverBudget(t *testing.T) {t.Fatalf("image-only turn lost or modified after over-budget fitting: %+v", last)}}++// TestFitMessages_RejectsTextOnlyTurnTrimmedToEmpty pins the validation rule: a+// multi-modal turn whose only part is text that the fitting trimmed away has+// nothing left to send, so the non-empty parts slice must not pass it as a valid+// user turn (Python's validate_fitted_messages rejects empty user content too).+func TestFitMessages_RejectsTextOnlyTurnTrimmedToEmpty(t *testing.T) {+ // The system share absorbs the whole (small) budget, so the user turn is+ // trimmed to nothing. A zero budget would be clamped to 8192 by the fitter.+ msgs := []schema.Message{+ {Role: schema.System, Content: strings.Repeat("sys ", 100)},+ {Role: schema.User, UserInputMultiContent: []schema.MessageInputPart{+ {Type: schema.ChatMessagePartTypeText, Text: strings.Repeat("question ", 400)},+ }},+ }+ fitted, fitErr := FitMessages("", msgs, 60)+ if fitErr == "" {+ t.Fatalf("an empty user turn was accepted: %+v", fitted)+ }+ if !strings.Contains(fitErr, "user message is empty") {+ t.Errorf("fit error = %q, want the empty-user-message error", fitErr)+ }+}++// TestFitMessages_KeepsTurnWithImageWhenTextTrimsToEmpty is the counterpart: the+// text part is trimmed away but the turn still carries an image payload, which+// IS usable — so it stays a valid user turn and the image is preserved.+func TestFitMessages_KeepsTurnWithImageWhenTextTrimsToEmpty(t *testing.T) {+ imgURL := "data:image/png;base64,AAAA"+ msgs := []schema.Message{+ {Role: schema.System, Content: strings.Repeat("sys ", 100)},+ {Role: schema.User, UserInputMultiContent: []schema.MessageInputPart{+ {Type: schema.ChatMessagePartTypeText, Text: strings.Repeat("question ", 400)},+ {Type: schema.ChatMessagePartTypeImageURL, Image: &schema.MessageInputImage{+ MessagePartCommon: schema.MessagePartCommon{URL: &imgURL},+ }},+ }},+ }+ fitted, fitErr := FitMessages("", msgs, 60)+ if fitErr != "" {+ t.Fatalf("turn with an image payload was rejected: %s", fitErr)+ }+ last := fitted[len(fitted)-1]+ if len(last.UserInputMultiContent) != 2 || last.UserInputMultiContent[1].Type != schema.ChatMessagePartTypeImageURL {+ t.Fatalf("image part lost: %+v", last.UserInputMultiContent)+ }+ if got := strings.TrimSpace(last.UserInputMultiContent[0].Text); got != "" {+ t.Errorf("text part = %q, want it trimmed away by the zero budget", got)+ }+}++// TestFitMessages_PreservesAllMessageFields pins the deep copy: fitting rewrites+// only the text it touches, so the returned messages must still carry every+// other schema.Message field (tool calls, tool ids, reasoning, generated parts,+// extras, name) — and the caller's payload must not be aliased.+func TestFitMessages_PreservesAllMessageFields(t *testing.T) {+ imgURL := "data:image/png;base64,AAAA"+ msgs := []schema.Message{+ {Role: schema.System, Content: "sys"},+ {+ Role: schema.Assistant,+ Content: "calling a tool",+ ReasoningContent: "because",+ AssistantGenMultiContent: []schema.MessageOutputPart{+ {Type: schema.ChatMessagePartTypeText, Text: "generated"},+ },+ ToolCalls: []schema.ToolCall{{+ ID: "call-1", Type: "function",+ Function: schema.FunctionCall{Name: "rag", Arguments: "{}"},+ }},+ Extra: map[string]any{"k": "v"},+ },+ {Role: schema.Tool, Content: "tool result", ToolCallID: "call-1", ToolName: "rag"},+ {Role: schema.User, Name: "u", UserInputMultiContent: []schema.MessageInputPart{+ {Type: schema.ChatMessagePartTypeText, Text: "question"},+ {Type: schema.ChatMessagePartTypeImageURL, Image: &schema.MessageInputImage{+ MessagePartCommon: schema.MessagePartCommon{URL: &imgURL},+ }},+ }},+ }+ fitted, fitErr := FitMessages("", msgs, 100000)+ if fitErr != "" {+ t.Fatalf("unexpected fit error: %s", fitErr)+ }+ if len(fitted) != 4 {+ t.Fatalf("got %d messages, want 4", len(fitted))+ }++ assistant := fitted[1]+ if assistant.ReasoningContent != "because" || len(assistant.AssistantGenMultiContent) != 1 ||+ len(assistant.ToolCalls) != 1 || assistant.ToolCalls[0].ID != "call-1" ||+ assistant.ToolCalls[0].Function.Name != "rag" || assistant.Extra["k"] != "v" {+ t.Errorf("assistant message lost fields after fitting: %+v", assistant)+ }+ if tool := fitted[2]; tool.ToolCallID != "call-1" || tool.ToolName != "rag" {+ t.Errorf("tool message lost fields after fitting: %+v", tool)+ }+ user := fitted[3]+ if user.Name != "u" || len(user.UserInputMultiContent) != 2 {+ t.Errorf("user message lost fields after fitting: %+v", user)+ }+ img := user.UserInputMultiContent[1].Image+ if img == nil || img.URL == nil || *img.URL != imgURL {+ t.Fatalf("image payload lost after fitting: %+v", img)+ }+ if img.URL == &imgURL {+ t.Error("image url was not deep-copied: the fitted message aliases the caller's payload")+ }+}diff --git a/internal/agent/tool/agentic_search.go b/internal/agent/tool/agentic_search.goindex 98ab03eea3..4586662397 100644--- a/internal/agent/tool/agentic_search.go+++ b/internal/agent/tool/agentic_search.go@@ -285,7 +285,11 @@ func narrowContent(content string, kwds []string) (string, bool) {return "..." + highlightKeywords(b.String(), kwds) + "...", true}-// highlightKeywords wraps keyword occurrences in <em>.+// highlightKeywords stars keyword occurrences, mirroring Python+// _highlight_keywords' `*term*` marker (text_processing.py:412) — NOT an XML+// tag: search.py narrows through the same helper, so a starred span is what the+// Python canvas hands the model. (The <em> tags elsewhere in this port are the+// engine's highlight markup: rag/utils/*_conn.py, akshare.go.)func highlightKeywords(text string, kwds []string) string {if len(kwds) == 0 {return text@@ -310,7 +314,7 @@ func highlightKeywords(text string, kwds []string) string {}pattern += ")"re := regexp.MustCompile(`(?i)` + pattern)- return re.ReplaceAllString(text, "<em>${1}</em>")+ return re.ReplaceAllString(text, "*${1}*")}func md5Hex(s string) string {diff --git a/internal/agent/tool/agentic_search_test.go b/internal/agent/tool/agentic_search_test.goindex c0c49baf91..c8354a93ca 100644--- a/internal/agent/tool/agentic_search_test.go+++ b/internal/agent/tool/agentic_search_test.go@@ -33,7 +33,7 @@ func TestNarrowContent_KeepsKeywordSentence(t *testing.T) {if !strings.Contains(nc, "key insight") {t.Errorf("narrowed content missing keyword sentence: %q", nc)}- if !strings.Contains(nc, "<em>rocket</em>") {+ if !strings.Contains(nc, "*rocket*") {t.Errorf("narrowed content missing highlight: %q", nc)}}diff --git a/internal/agent/tool/dataset_navigation.go b/internal/agent/tool/dataset_navigation.goindex 191d27fc72..d302679c59 100644--- a/internal/agent/tool/dataset_navigation.go+++ b/internal/agent/tool/dataset_navigation.go@@ -39,8 +39,10 @@ type datasetNavigationArgs struct {Topic string `json:"topic"`Keywords string `json:"keywords,omitempty"`DatasetIDs []string `json:"dataset_ids,omitempty"`- DocScope string `json:"doc_scope,omitempty"`- MaxDocs int `json:"max_docs,omitempty"`+ // DocScope restricts the routed documents, mirroring Python's+ // dataset_navigation_by_tree(doc_scope: list[str] | None = None).+ DocScope []string `json:"doc_scope,omitempty"`+ MaxDocs int `json:"max_docs,omitempty"`}// datasetNavigationResult is the JSON shape returned to the model.@@ -90,6 +92,10 @@ func (d *DatasetNavigationByTree) Info(_ context.Context) (*schema.ToolInfo, errType: schema.String,Desc: "Optional additional keywords to disambiguate the topic.",},+ "doc_scope": {+ Type: schema.Array,+ Desc: "Optional doc ids to restrict the navigation to.",+ },}),}, nil}@@ -133,10 +139,30 @@ func (d *DatasetNavigationByTree) InvokableRun(ctx context.Context, argumentsInJ// Route RELEVANT docs by querying the nav tree with the topic (semantic KNN).// The topic is the routing signal — we must not return arbitrary doc ids.query := strings.TrimSpace(args.Topic + " " + args.Keywords)++ // Honor the supplied doc scope. Python's dataset_navigation_by_tree threads+ // tools.scoped_doc_ids(doc_scope) into BOTH its tree walk and its+ // content-recall fallback; the canvas context carries no session scope, so+ // only the caller-supplied scope applies here. It is enforced in collect()+ // rather than only at the Search call, because the cluster-walk fallback's+ // ListClusters/ListChildren take no scope argument.+ docScope := compactStrings(args.DocScope)+ scopeSet := make(map[string]struct{}, len(docScope))+ for _, id := range docScope {+ scopeSet[id] = struct{}{}+ }+ inScope := func(id string) bool {+ if len(scopeSet) == 0 {+ return true+ }+ _, ok := scopeSet[id]+ return ok+ }+seen := map[string]struct{}{}var docs []stringcollect := func(id string) {- if id == "" {+ if id == "" || !inScope(id) {return}if _, ok := seen[id]; ok {@@ -149,12 +175,11 @@ func (d *DatasetNavigationByTree) InvokableRun(ctx context.Context, argumentsInJdocs = append(docs, id)}- // Primary: semantic search over each dataset's nav tree.+ // Primary: semantic search over each dataset's nav tree. The scope is+ // forwarded so the service also trims each cluster's returned coverage to it+ // (a cluster that merely OVERLAPS the scope must not surface extra docs).for _, datasetID := range datasetIDs {- // No doc scope: Python's dataset_navigation_by_tree applies- // tools.scoped_doc_ids(None), but this canvas tool's context carries no- // session document scope, so the read stays dataset-wide.- hits, err := ns.Search(ctx, tenantID, datasetID, query, nil, nil, maxDocs)+ hits, err := ns.Search(ctx, tenantID, datasetID, query, nil, docScope, maxDocs)if err != nil {continue}@@ -170,7 +195,9 @@ func (d *DatasetNavigationByTree) InvokableRun(ctx context.Context, argumentsInJ}// Fallback: if semantic routing found nothing (e.g. no embedder), walk the- // root clusters so the tool still returns a useful (if coarse) doc set.+ // root clusters so the tool still returns a useful (if coarse) doc set. The+ // scope still applies — collect() filters these leaves, standing in for+ // Python's _content_recall_docs(tools, query, doc_scope).if len(docs) == 0 {for _, datasetID := range datasetIDs {clusters, _, err := ns.ListClusters(ctx, tenantID, datasetID, 0, 100)@@ -205,9 +232,22 @@ func (d *DatasetNavigationByTree) InvokableRun(ctx context.Context, argumentsInJ}func (d *DatasetNavigationByTree) mergeDefaults(args datasetNavigationArgs) datasetNavigationArgs {+ // Blank request values count as NOT SUPPLIED, and they must be compacted+ // before the default is considered: a request like {"doc_scope":[" "]} has a+ // non-zero length, so it used to suppress d.defaults.DocScope and then+ // compact away at the use site — leaving an empty scope, which inScope() and+ // ns.Search read as "unscoped". The configured restriction was therefore+ // silently disabled by a value that carries no document id, letting the tool+ // return documents the default scope excludes. Python's+ // RAGTools.scoped_doc_ids treats a falsy request scope the same way this now+ // does: fall back to the configured scope (agentic_rag.py:352-358).+ args.DocScope = compactStrings(args.DocScope)if len(args.DatasetIDs) == 0 && len(d.defaults.DatasetIDs) != 0 {args.DatasetIDs = append([]string(nil), d.defaults.DatasetIDs...)}+ if len(args.DocScope) == 0 && len(d.defaults.DocScope) != 0 {+ args.DocScope = compactStrings(d.defaults.DocScope)+ }if args.MaxDocs <= 0 {args.MaxDocs = d.defaults.MaxDocs}diff --git a/internal/agent/tool/dataset_navigation_test.go b/internal/agent/tool/dataset_navigation_test.goindex 75d7ad8821..929378ab98 100644--- a/internal/agent/tool/dataset_navigation_test.go+++ b/internal/agent/tool/dataset_navigation_test.go@@ -2,6 +2,7 @@ package toolimport ("context"+ "encoding/json""sync""testing"@@ -14,7 +15,8 @@ import (// queries by topic rather than walking arbitrary clusters.type navRoutingFake struct {mu sync.Mutex- searched []string // topics passed to Search+ searched []string // topics passed to Search+ scopes [][]string // doc scopes passed to Searchhits []nav.NavHitclusters []nav.NavNodechildren map[string][]nav.NavNode@@ -24,9 +26,10 @@ func (f *navRoutingFake) UpsertDoc(context.Context, nav.UpsertDocInput) error {func (f *navRoutingFake) RemoveDoc(context.Context, string, string, string) error {return nil}-func (f *navRoutingFake) Search(_ context.Context, _, _ string, query string, _ []float32, _ []string, _ int) ([]nav.NavHit, error) {+func (f *navRoutingFake) Search(_ context.Context, _, _ string, query string, _ []float32, docScope []string, _ int) ([]nav.NavHit, error) {f.mu.Lock()f.searched = append(f.searched, query)+ f.scopes = append(f.scopes, append([]string(nil), docScope...))f.mu.Unlock()return f.hits, nil}@@ -45,6 +48,14 @@ func (f *navRoutingFake) searchedTopics() []string {return append([]string(nil), f.searched...)}+func (f *navRoutingFake) searchedScopes() [][]string {+ f.mu.Lock()+ defer f.mu.Unlock()+ out := make([][]string, len(f.scopes))+ copy(out, f.scopes)+ return out+}+// TestDatasetNavigation_UsesTopicRouting asserts the router queries the nav tree// with the topic (semantic search), rather than blindly walking clusters and// returning arbitrary doc ids.@@ -123,6 +134,104 @@ func TestCanvasDatasetIDs_DedupEmpty(t *testing.T) {}}+func navTestContext(t *testing.T) context.Context {+ t.Helper()+ state := runtime.NewCanvasState("run-1", "task-1")+ state.Sys["tenant_id"] = "tenant-1"+ return runtime.WithState(t.Context(), state)+}++func decodeNavDocs(t *testing.T, out string) []string {+ t.Helper()+ var res struct {+ Docs []string `json:"docs"`+ }+ if err := json.Unmarshal([]byte(out), &res); err != nil {+ t.Fatalf("unmarshal result %q: %v", out, err)+ }+ return res.Docs+}++// TestDatasetNavigation_HonorsDocScope pins that a supplied doc_scope reaches the+// nav service AND filters the routed result: Python's dataset_navigation_by_tree+// threads tools.scoped_doc_ids(doc_scope), so an out-of-scope document must never+// come back from a scoped request.+func TestDatasetNavigation_HonorsDocScope(t *testing.T) {+ fake := &navRoutingFake{hits: []nav.NavHit{+ {Type: nav.TypeNavDoc, DocID: "d1", Name: "in scope"},+ {Type: nav.TypeNavDoc, DocID: "out", Name: "out of scope"},+ }}+ prev := nav.GetNavService()+ nav.SetNavService(fake)+ defer func() { nav.SetNavService(prev) }()++ tool := NewDatasetNavigationByTree()+ out, err := tool.InvokableRun(navTestContext(t), `{"topic":"X","dataset_ids":["kb1"],"doc_scope":["d1"]}`)+ if err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ scopes := fake.searchedScopes()+ if len(scopes) != 1 || len(scopes[0]) != 1 || scopes[0][0] != "d1" {+ t.Fatalf("doc scope not forwarded to Search: %v", scopes)+ }+ if docs := decodeNavDocs(t, out); len(docs) != 1 || docs[0] != "d1" {+ t.Errorf("docs = %v, want only the in-scope d1", docs)+ }+}++// TestDatasetNavigation_DocScopeFiltersClusterFallback pins the fallback half:+// when semantic routing finds nothing, the cluster walk must still not surface+// out-of-scope documents (ListClusters/ListChildren take no scope, so the tool+// filters the leaves itself — standing in for Python's _content_recall_docs).+func TestDatasetNavigation_DocScopeFiltersClusterFallback(t *testing.T) {+ fake := &navRoutingFake{+ clusters: []nav.NavNode{{Name: "c1"}},+ children: map[string][]nav.NavNode{+ "c1": {{DocID: "d1"}, {DocID: "out"}},+ },+ }+ prev := nav.GetNavService()+ nav.SetNavService(fake)+ defer func() { nav.SetNavService(prev) }()++ tool := NewDatasetNavigationByTree()+ out, err := tool.InvokableRun(navTestContext(t), `{"topic":"X","dataset_ids":["kb1"],"doc_scope":["d1"]}`)+ if err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ if docs := decodeNavDocs(t, out); len(docs) != 1 || docs[0] != "d1" {+ t.Errorf("fallback docs = %v, want only the in-scope d1", docs)+ }+}++// TestDatasetNavigation_BlankRequestScopeFallsBackToDefaults pins that a blank+// request value cannot DISABLE the configured default scope: [" "] has a+// non-zero length, so it suppressed the default and then compacted to an empty+// scope, which inScope()/ns.Search read as "unscoped" — the tool returned+// documents the default scope excludes.+func TestDatasetNavigation_BlankRequestScopeFallsBackToDefaults(t *testing.T) {+ fake := &navRoutingFake{hits: []nav.NavHit{+ {Type: nav.TypeNavDoc, DocID: "d1", Name: "in scope"},+ {Type: nav.TypeNavDoc, DocID: "out", Name: "out of scope"},+ }}+ prev := nav.GetNavService()+ nav.SetNavService(fake)+ defer func() { nav.SetNavService(prev) }()++ tool := NewDatasetNavigationByTreeWithDefaults(datasetNavigationArgs{DocScope: []string{"d1"}})+ out, err := tool.InvokableRun(navTestContext(t), `{"topic":"X","dataset_ids":["kb1"],"doc_scope":[" "]}`)+ if err != nil {+ t.Fatalf("InvokableRun: %v", err)+ }+ scopes := fake.searchedScopes()+ if len(scopes) != 1 || len(scopes[0]) != 1 || scopes[0][0] != "d1" {+ t.Fatalf("doc scope forwarded to Search = %v, want the configured default [d1]", scopes)+ }+ if docs := decodeNavDocs(t, out); len(docs) != 1 || docs[0] != "d1" {+ t.Errorf("docs = %v, want only the in-scope d1 (a blank request value must not disable the default scope)", docs)+ }+}+func containsStr(s, sub string) bool {return len(s) > 0 && len(sub) > 0 && (s == sub || containsSub(s, sub))}diff --git a/internal/entity/models/aliyun.go b/internal/entity/models/aliyun.goindex 471ba55468..388dc3170d 100644--- a/internal/entity/models/aliyun.go+++ b/internal/entity/models/aliyun.go@@ -341,7 +341,10 @@ type aliyunNativeEmbedResponse struct {RequestID string `json:"request_id"`Output struct {Embeddings []struct {- TextIndex int `json:"text_index"`+ // TextIndex is a pointer so an absent/null key is distinguishable+ // from index 0: Python does embds[e["text_index"]] and raises+ // KeyError when the key is missing (wrapped into EmbeddingError).+ TextIndex *int `json:"text_index"`Embedding []float64 `json:"embedding"`} `json:"embeddings"`} `json:"output"`@@ -355,6 +358,16 @@ type aliyunNativeEmbedResponse struct {// Python QWenEmbed). Inputs are sent in batches of 4 like Python; the response's// text_index is relative to the batch, so it is offset back to the caller's// slice before being returned.+//+// The response is reassembled exactly like Python QWenEmbed.encode: each batch's+// chunk is sized by the NUMBER OF RETURNED EMBEDDINGS and every vector is+// PLACED at its batch-relative text_index (never appended in response order),+// because consumers such as NavEmbedder read the slice positionally and discard+// EmbeddingData.Index. As in Python, a duplicate text_index overwrites the+// earlier vector, a gap leaves an empty vector in that slot, and a text_index+// outside the returned chunk raises — a NEGATIVE text_index counts from the end+// of the chunk (embds[-1] is the last slot), exactly like a Python list+// assignment.func (a *AliyunModel) embedNative(ctx context.Context, root, modelName string, request EmbedRequest, apiConfig *APIConfig, modelUsage *common.ModelUsage) ([]EmbeddingData, error) {textType := "document"if request.Query {@@ -407,15 +420,32 @@ func (a *AliyunModel) embedNative(ctx context.Context, root, modelName string, rif parsed.Code != "" {return nil, fmt.Errorf("aliyun native embeddings API error: %s: %s", parsed.Code, parsed.Message)}+ // Mirror Python QWenEmbed.encode:+ // embds = [[] for _ in range(len(resp["output"]["embeddings"]))]+ // for e in resp["output"]["embeddings"]:+ // embds[e["text_index"]] = e["embedding"]+ // res.extend(embds)+ // including Python list-assignment indexing: a negative text_index+ // counts from the END of the chunk (embds[-1] is the last slot), and out+ // of range in either direction raises.+ embds := make([]EmbeddingData, len(parsed.Output.Embeddings))for _, item := range parsed.Output.Embeddings {- if item.TextIndex < 0 || item.TextIndex >= len(batch) {- return nil, fmt.Errorf("aliyun native embeddings response index %d out of range for batch of %d", item.TextIndex, len(batch))+ if item.TextIndex == nil {+ return nil, fmt.Errorf("aliyun native embeddings response item missing text_index")+ }+ idx := *item.TextIndex+ if idx < 0 {+ idx += len(embds)}- embeddings = append(embeddings, EmbeddingData{+ if idx < 0 || idx >= len(embds) {+ return nil, fmt.Errorf("aliyun native embeddings response index %d out of range for %d embeddings", *item.TextIndex, len(embds))+ }+ embds[idx] = EmbeddingData{Embedding: item.Embedding,- Index: start + item.TextIndex,- })+ Index: len(embeddings) + idx,+ }}+ embeddings = append(embeddings, embds...)recordResponseUsage(modelUsage, parsed.RequestID, &TokenUsage{TotalTokens: parsed.Usage.TotalTokens}, "embedding")}return embeddings, nildiff --git a/internal/entity/models/aliyun_test.go b/internal/entity/models/aliyun_test.goindex 6a962220cd..41b3e308d3 100644--- a/internal/entity/models/aliyun_test.go+++ b/internal/entity/models/aliyun_test.go@@ -738,3 +738,200 @@ func TestAliyunEmbedNativeSendsTextTypeAndBatches(t *testing.T) {t.Errorf("query text_type = %v, want query", bodies[len(bodies)-1]["parameters"])}}++// TestAliyunEmbedNativePlacesVectorsByTextIndex pins that an out-of-order+// response is reordered onto the caller's inputs: nav_embedder/dataset-utils+// read the result positionally and discard Index, so placing by text_index is+// what keeps each vector bound to the right text.+func TestAliyunEmbedNativePlacesVectorsByTextIndex(t *testing.T) {+ withSSRFBypass(t)+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ // Deliberately reversed: text_index 3,2,1,0 with distinct vectors.+ _, _ = w.Write([]byte(`{"output":{"embeddings":[` ++ `{"text_index":3,"embedding":[3]},` ++ `{"text_index":2,"embedding":[2]},` ++ `{"text_index":1,"embedding":[1]},` ++ `{"text_index":0,"embedding":[0]}` ++ `]},"usage":{"total_tokens":1}}`))+ }))+ defer srv.Close()++ m := NewAliyunModel(map[string]string{"default": srv.URL}, URLSuffix{Embedding: "embeddings"})+ apiKey := "test-key"+ got, err := m.embedNative(t.Context(), srv.URL, "text-embedding-v4",+ EmbedRequest{Texts: []string{"a", "b", "c", "d"}}, &APIConfig{ApiKey: &apiKey}, nil)+ if err != nil {+ t.Fatalf("embedNative: %v", err)+ }+ if len(got) != 4 {+ t.Fatalf("len = %d, want 4", len(got))+ }+ for i, e := range got {+ if e.Index != i || len(e.Embedding) != 1 || e.Embedding[0] != float64(i) {+ t.Errorf("got[%d] = {index:%d vec:%v}, want {index:%d vec:[%d]}", i, e.Index, e.Embedding, i, i)+ }+ }+}++// TestAliyunEmbedNativeDuplicateIndexLastWins pins Python QWenEmbed.encode's+// "embds[e['text_index']] = e['embedding']": a repeated text_index overwrites+// the earlier vector and the slot left unfilled stays empty — no error.+func TestAliyunEmbedNativeDuplicateIndexLastWins(t *testing.T) {+ withSSRFBypass(t)+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"output":{"embeddings":[` ++ `{"text_index":0,"embedding":[0]},` ++ `{"text_index":1,"embedding":[1]},` ++ `{"text_index":1,"embedding":[9]}` ++ `]},"usage":{"total_tokens":1}}`))+ }))+ defer srv.Close()++ m := NewAliyunModel(map[string]string{"default": srv.URL}, URLSuffix{Embedding: "embeddings"})+ apiKey := "test-key"+ got, err := m.embedNative(t.Context(), srv.URL, "text-embedding-v4",+ EmbedRequest{Texts: []string{"a", "b", "c"}}, &APIConfig{ApiKey: &apiKey}, nil)+ if err != nil {+ t.Fatalf("embedNative: %v", err)+ }+ if len(got) != 3 {+ t.Fatalf("len = %d, want 3 (one chunk entry per returned embedding)", len(got))+ }+ if len(got[0].Embedding) != 1 || got[0].Embedding[0] != 0 {+ t.Errorf("got[0] = %v, want [0]", got[0].Embedding)+ }+ if len(got[1].Embedding) != 1 || got[1].Embedding[0] != 9 {+ t.Errorf("got[1] = %v, want the LAST duplicate [9]", got[1].Embedding)+ }+ if len(got[2].Embedding) != 0 {+ t.Errorf("got[2] = %v, want the unfilled slot to stay empty", got[2].Embedding)+ }+}++// TestAliyunEmbedNativeShortResponseStaysShort pins that Go mirrors Python's+// len(resp.embeddings)-sized chunks: a response covering only part of the batch+// is NOT rejected, it just yields a shorter result (Python's res.extend).+func TestAliyunEmbedNativeShortResponseStaysShort(t *testing.T) {+ withSSRFBypass(t)+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"output":{"embeddings":[` ++ `{"text_index":0,"embedding":[0]},` ++ `{"text_index":1,"embedding":[1]}` ++ `]},"usage":{"total_tokens":1}}`))+ }))+ defer srv.Close()++ m := NewAliyunModel(map[string]string{"default": srv.URL}, URLSuffix{Embedding: "embeddings"})+ apiKey := "test-key"+ got, err := m.embedNative(t.Context(), srv.URL, "text-embedding-v4",+ EmbedRequest{Texts: []string{"a", "b", "c", "d"}}, &APIConfig{ApiKey: &apiKey}, nil)+ if err != nil {+ t.Fatalf("embedNative: %v", err)+ }+ if len(got) != 2 {+ t.Fatalf("len = %d, want 2 (response count, Python res.extend)", len(got))+ }+ if got[0].Embedding[0] != 0 || got[1].Embedding[0] != 1 {+ t.Errorf("got = %v, want [[0],[1]]", got)+ }+}++// TestAliyunEmbedNativeRejectsOutOfRangeIndex pins Python's IndexError: a+// text_index past the returned chunk is an error.+func TestAliyunEmbedNativeRejectsOutOfRangeIndex(t *testing.T) {+ withSSRFBypass(t)+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"output":{"embeddings":[` ++ `{"text_index":5,"embedding":[5]}` ++ `]},"usage":{"total_tokens":1}}`))+ }))+ defer srv.Close()++ m := NewAliyunModel(map[string]string{"default": srv.URL}, URLSuffix{Embedding: "embeddings"})+ apiKey := "test-key"+ _, err := m.embedNative(t.Context(), srv.URL, "text-embedding-v4",+ EmbedRequest{Texts: []string{"a", "b"}}, &APIConfig{ApiKey: &apiKey}, nil)+ if err == nil || !strings.Contains(err.Error(), "out of range") {+ t.Fatalf("err = %v, want out-of-range error", err)+ }+}++// TestAliyunEmbedNativeRejectsMissingTextIndexKey pins Python's KeyError on+// e["text_index"]: an item without the key is an error, not a silent index 0.+func TestAliyunEmbedNativeRejectsMissingTextIndexKey(t *testing.T) {+ withSSRFBypass(t)+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"output":{"embeddings":[` ++ `{"embedding":[1]}` ++ `]},"usage":{"total_tokens":1}}`))+ }))+ defer srv.Close()++ m := NewAliyunModel(map[string]string{"default": srv.URL}, URLSuffix{Embedding: "embeddings"})+ apiKey := "test-key"+ _, err := m.embedNative(t.Context(), srv.URL, "text-embedding-v4",+ EmbedRequest{Texts: []string{"a"}}, &APIConfig{ApiKey: &apiKey}, nil)+ if err == nil || !strings.Contains(err.Error(), "text_index") {+ t.Fatalf("err = %v, want missing-text_index error", err)+ }+}++// TestAliyunEmbedNativeNegativeIndexCountsFromEnd pins Python list-assignment+// semantics: a negative text_index indexes from the END of the chunk, so+// embds[-1] overwrites the last slot.+func TestAliyunEmbedNativeNegativeIndexCountsFromEnd(t *testing.T) {+ withSSRFBypass(t)+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"output":{"embeddings":[` ++ `{"text_index":-1,"embedding":[7]},` ++ `{"text_index":0,"embedding":[0]},` ++ `{"text_index":1,"embedding":[1]}` ++ `]},"usage":{"total_tokens":1}}`))+ }))+ defer srv.Close()++ m := NewAliyunModel(map[string]string{"default": srv.URL}, URLSuffix{Embedding: "embeddings"})+ apiKey := "test-key"+ got, err := m.embedNative(t.Context(), srv.URL, "text-embedding-v4",+ EmbedRequest{Texts: []string{"a", "b", "c"}}, &APIConfig{ApiKey: &apiKey}, nil)+ if err != nil {+ t.Fatalf("embedNative: %v", err)+ }+ if len(got) != 3 {+ t.Fatalf("len = %d, want 3", len(got))+ }+ // embds[-1] = [7] lands in the last slot: [[0],[1],[7]].+ want := []float64{0, 1, 7}+ for i, w := range want {+ if len(got[i].Embedding) != 1 || got[i].Embedding[0] != w {+ t.Errorf("got[%d] = %v, want [%v]", i, got[i].Embedding, w)+ }+ }+}++// TestAliyunEmbedNativeRejectsTooNegativeIndex pins that an index below+// -len(chunk) raises, exactly like Python's IndexError.+func TestAliyunEmbedNativeRejectsTooNegativeIndex(t *testing.T) {+ withSSRFBypass(t)+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"output":{"embeddings":[` ++ `{"text_index":-5,"embedding":[5]}` ++ `]},"usage":{"total_tokens":1}}`))+ }))+ defer srv.Close()++ m := NewAliyunModel(map[string]string{"default": srv.URL}, URLSuffix{Embedding: "embeddings"})+ apiKey := "test-key"+ _, err := m.embedNative(t.Context(), srv.URL, "text-embedding-v4",+ EmbedRequest{Texts: []string{"a", "b"}}, &APIConfig{ApiKey: &apiKey}, nil)+ if err == nil || !strings.Contains(err.Error(), "out of range") {+ t.Fatalf("err = %v, want out-of-range error", err)+ }+}diff --git a/internal/entity/models/chat_tools.go b/internal/entity/models/chat_tools.goindex 4149787827..c3ea0b1ddc 100644--- a/internal/entity/models/chat_tools.go+++ b/internal/entity/models/chat_tools.go@@ -162,7 +162,10 @@ func runToolLoop(ctx context.Context, cm *ChatModel, history []Message, toolsLisvar toolAnswer stringhistory, toolAnswer, hit = appendToolResults(history, resp.ToolCalls, cm.ToolConfig.ToolCallSession, terminalSet(cm))if hit {- addRoundUsage(resp)+ // This round's usage was already folded in by addRoundUsage above+ // (it accumulates into aggUsage and forwards a per-round delta to the+ // run-usage sink, which ADDS): calling it again here would count the+ // round twice in both totalTokens and the recorded run usage.return toolAnswer, totalTokens, nil}// history now carries this round's tool results; continue to the next round.diff --git a/internal/entity/models/chat_tools_test.go b/internal/entity/models/chat_tools_test.goindex 4cb98e8ffb..beaeb57efe 100644--- a/internal/entity/models/chat_tools_test.go+++ b/internal/entity/models/chat_tools_test.go@@ -16,7 +16,12 @@package models-import "testing"+import (+ "context"+ "testing"++ "ragflow/internal/tokenizer"+)// stubSession returns a canned result per tool name.type stubSession struct {@@ -170,3 +175,40 @@ func TestSendTerminalEmptyAnswerIsSilent(t *testing.T) {t.Fatalf("streamed %v, want nothing for an empty terminal answer", got)}}++// TestChatWithToolsCountsTerminalRoundUsageOnce pins that a terminal-tool round+// contributes its tokens exactly once, in both the returned total and the+// run-usage sink (whose Add accumulates): runToolLoop folds the round in before+// the short-circuit decides.+func TestChatWithToolsCountsTerminalRoundUsageOnce(t *testing.T) {+ const (+ answer = "[SUMMARY]"+ total = 15+ )+ driver := &captureToolDriver{resp: &ChatResponse{+ ToolCalls: []map[string]interface{}{toolCallMsg("call-1", "summarize_document", `{"doc_id":"abc"}`)},+ Usage: &TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: total},+ }}+ modelName, apiKey := "m", "k"+ cm := NewChatModel(driver, &modelName, &APIConfig{ApiKey: &apiKey})+ cm.ToolConfig = &ToolConfig{+ Tools: `[{"type":"function","function":{"name":"summarize_document"}}]`,+ ToolCallSession: &stubSession{results: map[string]string{"summarize_document": answer}},+ TerminalTools: map[string]struct{}{"summarize_document": {}},+ }++ ctx := tokenizer.WithRunUsage(context.Background())+ got, tokens, err := cm.ChatWithTools(ctx, "", []Message{{Role: "user", Content: "q"}}, &ChatConfig{})+ if err != nil {+ t.Fatalf("ChatWithTools: %v", err)+ }+ if got != answer {+ t.Fatalf("answer = %q, want the terminal result %q", got, answer)+ }+ if tokens != total {+ t.Errorf("totalTokens = %d, want %d (the terminal round must be counted once)", tokens, total)+ }+ if _, _, runTotal, calls := tokenizer.GetRunUsage(ctx).Snapshot(); runTotal != total || calls != 1 {+ t.Errorf("run usage total=%d calls=%d, want total=%d calls=1", runTotal, calls, total)+ }+}diff --git a/internal/entity/models/llm.go b/internal/entity/models/llm.goindex 16e3e50e33..7ec073e960 100644--- a/internal/entity/models/llm.go+++ b/internal/entity/models/llm.go@@ -262,8 +262,6 @@ func (m *EinoChatModel) chatConfigForGenerate() (*ChatConfig, error) {choice = "required"break}- //if m.toolChoice != nil {- // choice = *m.toolChoice}cfg.ToolChoice = &choicefor _, tool := range m.tools {@@ -275,9 +273,36 @@ func (m *EinoChatModel) chatConfigForGenerate() (*ChatConfig, error) {break}}+ // An explicit WithToolChoice overrides both defaults above; the setter+ // documents that the choice reaches the driver through this config. A+ // keyword choice leaves ToolChoiceValue as an untyped nil: the driver only+ // falls back to the plain string when the field is nil (base_model.go:565),+ // and a typed-nil map would be sent as `"tool_choice": null`.+ if m.toolChoice != nil && *m.toolChoice != "" {+ cfg.ToolChoice = m.toolChoice+ cfg.ToolChoiceValue = nil+ if body := toolChoiceBody(*m.toolChoice); body != nil {+ cfg.ToolChoiceValue = body+ }+ }return cfg, nil}+// toolChoiceBody returns the OpenAI tool_choice OBJECT form for a choice naming+// a specific tool, and nil for the keyword forms ("auto"/"none"/"required"),+// which travel as the plain ToolChoice string (base_model.go:561-568 prefers+// this value whenever it is set).+func toolChoiceBody(choice string) map[string]any {+ switch choice {+ case "auto", "none", "required":+ return nil+ }+ return map[string]any{+ "type": "function",+ "function": map[string]any{"name": choice},+ }+}+func openAIToolsFromEino(infos []*schema.ToolInfo) ([]map[string]any, error) {tools := make([]map[string]any, 0, len(infos))for _, info := range infos {diff --git a/internal/entity/models/llm_test.go b/internal/entity/models/llm_test.goindex a3a150b33b..7200cb32c2 100644--- a/internal/entity/models/llm_test.go+++ b/internal/entity/models/llm_test.go@@ -5,6 +5,7 @@ import ("errors""io""ragflow/internal/common"+ "reflect""testing""github.com/cloudwego/eino/schema"@@ -50,6 +51,85 @@ func TestEinoChatModelAllowsFinalAnswerAfterToolResult(t *testing.T) {}}+// TestEinoChatModelAppliesExplicitToolChoice pins that WithToolChoice actually+// reaches the driver's configuration (its doc promises exactly that): a keyword+// choice travels as the plain string with no object value, a named tool travels+// in the OpenAI object form, and either overrides the execute_code default.+func TestEinoChatModelAppliesExplicitToolChoice(t *testing.T) {+ cases := []struct {+ name string+ tools []*schema.ToolInfo+ choice string+ wantChoice string+ wantValue map[string]any // nil for the keyword forms+ }{+ {+ name: "keyword none overrides the execute_code default",+ tools: []*schema.ToolInfo{{Name: "execute_code"}},+ choice: "none",+ wantChoice: "none",+ },+ {+ name: "explicit required without execute_code",+ tools: []*schema.ToolInfo{{Name: "rag"}},+ choice: "required",+ wantChoice: "required",+ },+ {+ name: "named tool uses the object form",+ tools: []*schema.ToolInfo{{Name: "rag"}, {Name: "summarize_document"}},+ choice: "summarize_document",+ wantChoice: "summarize_document",+ wantValue: map[string]any{+ "type": "function",+ "function": map[string]any{"name": "summarize_document"},+ },+ },+ {+ name: "no explicit choice keeps the execute_code default",+ tools: []*schema.ToolInfo{{Name: "execute_code"}},+ wantChoice: "required",+ wantValue: map[string]any{+ "type": "function",+ "function": map[string]any{"name": "execute_code"},+ },+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ modelName := "chat"+ model := NewEinoChatModel(NewChatModel(&captureToolDriver{}, &modelName, &APIConfig{}), nil)+ bound, err := model.WithTools(tc.tools)+ if err != nil {+ t.Fatalf("WithTools: %v", err)+ }+ wrapper := bound.(*EinoChatModel)+ if tc.choice != "" {+ wrapper = wrapper.WithToolChoice(tc.choice)+ }+ cfg, err := wrapper.chatConfigForGenerate()+ if err != nil {+ t.Fatalf("chatConfigForGenerate: %v", err)+ }+ if cfg.ToolChoice == nil {+ t.Fatalf("ToolChoice = nil, want %q", tc.wantChoice)+ }+ if *cfg.ToolChoice != tc.wantChoice {+ t.Fatalf("ToolChoice = %q, want %q", *cfg.ToolChoice, tc.wantChoice)+ }+ if tc.wantValue == nil {+ if cfg.ToolChoiceValue != nil {+ t.Errorf("ToolChoiceValue = %#v, want nil for a keyword choice", cfg.ToolChoiceValue)+ }+ return+ }+ if got, ok := cfg.ToolChoiceValue.(map[string]any); !ok || !reflect.DeepEqual(got, tc.wantValue) {+ t.Errorf("ToolChoiceValue = %#v, want %#v", cfg.ToolChoiceValue, tc.wantValue)+ }+ })+ }+}+func TestEinoChatModelStreamFiltersDoneSentinel(t *testing.T) {modelName := "chat"driver := &streamSentinelDriver{captureToolDriver: &captureToolDriver{}}diff --git a/internal/ingestion/component/chunker/title_cap.go b/internal/ingestion/component/chunker/title_cap.goindex a3da43b66b..b049c66160 100644--- a/internal/ingestion/component/chunker/title_cap.go+++ b/internal/ingestion/component/chunker/title_cap.go@@ -42,9 +42,11 @@ var (trimToTokenLimit = realTrimToTokenLimit)-// titleTokenCount counts tokens for text. numTokensFromString returns 0 when-// the encoder is unavailable; in that case fall back to the rune count so the-// cap is still enforced (mirrors Python #18455's character-count fallback).+// titleTokenCount counts tokens for text. tokenizer.NumTokensFromString now+// PANICS when the encoder is unavailable (mirroring Python's raise), so a 0 here+// can only come from the counter itself (empty input, or the stubbed counter the+// tests install); fall back to the rune count so the cap is still enforced+// (mirrors Python #18455's character-count fallback).func titleTokenCount(text string) int {if text == "" {return 0diff --git a/internal/rag/advanced_rag/agentic_rag.go b/internal/rag/advanced_rag/agentic_rag.goindex 35c8673f31..3e8cb71b2b 100644--- a/internal/rag/advanced_rag/agentic_rag.go+++ b/internal/rag/advanced_rag/agentic_rag.go@@ -854,7 +854,7 @@ type RAGCache struct {mu sync.Mutexentries map[string]ragCacheEntrylastVerdict string- // ConsecutiveUnanswerable mirrors Python RAGTools._consecutive_unanswerable+ // consecutiveUnanswerable mirrors Python RAGTools._consecutive_unanswerable// (agentic_rag.py:818): how many consecutive rag() calls ended without a// satisfying verdict. After two in a row, RAGTools.rag appends a// "[Research status] … STOP calling rag again" note to the answer so the@@ -862,7 +862,36 @@ type RAGCache struct {// (rebuilt per turn), so it counts consecutive insufficient rounds within a// single request; it is not persisted across turns unless a caller injects a// long-lived *RAGCache via RAGTools.Cache.- ConsecutiveUnanswerable int+ //+ // Private and guarded by mu: a turn's concurrent rag() calls share ONE+ // *RAGCache (Rag builds/stores it on deps.Cache before the outer react+ // loop), so an unsynchronized counter would race.+ consecutiveUnanswerable int+}++// NoteUnanswerable records one research round's verdict on the shared cache.+func (c *RAGCache) NoteUnanswerable(verdict string) {+ if c == nil {+ return+ }+ c.mu.Lock()+ defer c.mu.Unlock()+ if verdict == VerdictSufficient {+ c.consecutiveUnanswerable = 0+ } else {+ c.consecutiveUnanswerable+++ }+}++// ConsecutiveUnanswerable reports how many consecutive rag() rounds ended+// without a satisfying verdict.+func (c *RAGCache) ConsecutiveUnanswerable() int {+ if c == nil {+ return 0+ }+ c.mu.Lock()+ defer c.mu.Unlock()+ return c.consecutiveUnanswerable}type ragCacheEntry struct {@@ -930,7 +959,7 @@ func researchStatusTrailer(cache *RAGCache, resp *RunResponse) string {if resp.Verdict != VerdictInsufficient || resp.SCAFeedback == "" || resp.Answer == "" {return ""}- if cache != nil && cache.ConsecutiveUnanswerable >= 2 {+ if cache != nil && cache.ConsecutiveUnanswerable() >= 2 {return " STOP calling rag again: the same gaps remain."}return " If these gaps are material, call rag again with a question focused on them."@@ -1655,6 +1684,9 @@ func runOuterReact(ctx context.Context, deps RAGTools, req harness.RunRequest, lcomposeFinalAnswer(ctx, deps, req, p.kb, p.resp, logger)return p.resp}+ // The fold returned the LOWEST-INDEX terminal rag call's answer: keep that+ // call's evidence so the answer's [ID:n] markers line up with the chunks.+ session.selectEvidence(answer)p.resp.Answer = answerp.resp.Chunks = p.kb.Chunksp.resp.DocAggs = p.kb.DocAggs@@ -1718,6 +1750,9 @@ func runOuterReactStream(ctx context.Context, deps RAGTools, req harness.RunRequif p.resp.Answer == "" && !mux.terminalFired {p.resp.Answer = strings.TrimSpace(mux.outerText.String())}+ // Keep only the evidence of the call that owns the answer (selectEvidence+ // no-ops when the answer came from the outer model instead of a rag call).+ session.selectEvidence(p.resp.Answer)p.resp.Chunks = p.kb.Chunksp.resp.DocAggs = p.kb.DocAggsp.resp.EmptyResult = len(p.kb.Chunks) == 0@@ -1737,38 +1772,67 @@ type outerReactSession struct {// mux is set only on the streaming path; it merges the inner run's output// into the caller's sink.mux *outerStreamMux+ // mu guards the per-call result merge (publish) and the recorded per-call+ // outcomes (calls): appendToolResults invokes this session's tool calls+ // concurrently.+ mu sync.Mutex+ // calls records each rag call's answer + evidence pool in completion order so+ // selectEvidence can match the terminal fold's winning answer back to its own+ // pool. Guarded by mu.+ calls []ragCallResult}// ToolCall routes the two outer tools to their Go implementations.+//+// A round's tool calls arrive CONCURRENTLY — models.appendToolResults runs one+// goroutine per call — exactly as Python gathers them (asyncio.gather over the+// round's tool_calls, chat_model.py:670/:2555). Each rag call therefore runs on+// its own request/evidence/response and publishes the outcome under mu; sharing+// the session's would let two calls mix questions, evidence and answers.func (s *outerReactSession) ToolCall(name string, arguments map[string]interface{}) (string, error) {switch name {case "rag":+ // Work on a COPY of the request: this call must not rewrite the+ // session's question/images, which a concurrent call is reading.+ req := s.reqif q, ok := arguments["question"].(string); ok && q != "" {- s.req.Question = q+ req.Question = q}// Python's inner _compose_answer_from_evidence is text-only: the outer// model already saw the images via multimodal history, and the rephrased// question carries no images. Drop the original images so the inner// compose model (ComposeAnswerWith via composeFinalAnswer) does not// receive them — matching Python's inner compose.- s.req.Images = nil+ req.Images = nil++ // Per-call evidence, response and the projections that point at them:+ // Python gives every rag() invocation its own graph state and shares only+ // the tools object (agentic_rag.py:877 run_agentic_rag(self, messages)).+ kb := &harness.Kbinfos{}+ resp := &RunResponse{Mode: s.spec, Kbinfos: kb}+ sd := s.sd+ sd.KB = kb+ inner := s.deps+ inner.KB = kb+if s.mux != nil {// Streaming: s.deps already routes the inner research log and the// composed answer into the caller's sink, so return "" — the// terminal short-circuit stops the loop and sendTerminal stays// silent, so the answer is not streamed twice.s.mux.markTerminal()- RunAgenticRAG(s.ctx, s.deps, s.req, s.sd, s.kb, s.resp, s.logger, s.spec)- composeFinalAnswer(s.ctx, s.deps, s.req, s.kb, s.resp, s.logger)+ RunAgenticRAG(s.ctx, inner, req, sd, kb, resp, s.logger, s.spec)+ composeFinalAnswer(s.ctx, inner, req, kb, resp, s.logger)+ s.publish(resp, kb)return "", nil}// Non-streaming: disable the sink so the answer is not emitted twice,- // and hand it back for the terminal short-circuit to return as final.- inner := s.deps+ // and hand this call's own answer back for the terminal short-circuit.inner.AnswerSink = nil- RunAgenticRAG(s.ctx, inner, s.req, s.sd, s.kb, s.resp, s.logger, s.spec)- composeFinalAnswer(s.ctx, inner, s.req, s.kb, s.resp, s.logger)- return s.resp.Answer, nil+ RunAgenticRAG(s.ctx, inner, req, sd, kb, resp, s.logger, s.spec)+ composeFinalAnswer(s.ctx, inner, req, kb, resp, s.logger)+ s.publish(resp, kb)+ return resp.Answer, nilcase "summarize_document":docID, _ := arguments["doc_id"].(string)if docID == "" {@@ -1783,6 +1847,80 @@ func (s *outerReactSession) ToolCall(name string, arguments map[string]interfacereturn fmt.Sprintf("unknown tool %q", name), nil}+// publish merges one rag call's outcome into the session state. The inner run+// used to write that state directly, before each call got its own; the merge+// reproduces what a single call contributed, and for concurrent calls it keeps+// the FIRST non-empty answer/verdict (appendToolResults folds the first terminal+// hit in call order) while the evidence is unioned, so the caller still sees the+// chunks the answer was composed from.+func (s *outerReactSession) publish(call *RunResponse, kb *harness.Kbinfos) {+ s.mu.Lock()+ defer s.mu.Unlock()+ // Remember this call's own answer + evidence so selectEvidence can restore+ // the pool the winning answer was composed from.+ s.calls = append(s.calls, ragCallResult{answer: call.Answer, kb: kb})+ if s.resp.Answer == "" {+ s.resp.Answer = call.Answer+ }+ if s.resp.Verdict == "" {+ s.resp.Verdict = call.Verdict+ }+ if s.resp.SCAFeedback == "" {+ s.resp.SCAFeedback = call.SCAFeedback+ }+ if s.resp.CollectedAnswer == "" {+ s.resp.CollectedAnswer = call.CollectedAnswer+ }+ if len(s.resp.Slots) == 0 {+ s.resp.Slots = call.Slots+ }+ s.resp.Partial = s.resp.Partial || call.Partial+ s.kb.Chunks = append(s.kb.Chunks, kb.Chunks...)+ s.kb.DocAggs = append(s.kb.DocAggs, kb.DocAggs...)+ s.kb.Memory = append(s.kb.Memory, kb.Memory...)+ if s.kb.PreSummary == "" {+ s.kb.PreSummary = kb.PreSummary+ }+}++// ragCallResult is one rag call's outcome: the answer it composed and the+// evidence pool that produced it.+type ragCallResult struct {+ answer string+ kb *harness.Kbinfos+}++// selectEvidence leaves only the evidence of the rag call whose answer the+// terminal fold returned. The fold picks the LOWEST-INDEX terminal call while+// publish sees completion order, so the union alone can leave that answer's+// [ID:n] markers pointing into another call's chunks — the answer and the+// reference list would disagree. No-op when no recorded call matches (the+// terminal tool was summarize_document, or the answer did not come from a rag+// call), in which case the union stands.+//+// Matching is by exact answer text, so two calls that produce byte-identical+// answers are indistinguishable and the FIRST recorded (first to finish) one+// wins. That collision is accepted: plumbing the tool-call index through+// ToolCallSession is not worth it for a case that cannot happen with distinct+// questions.+func (s *outerReactSession) selectEvidence(answer string) {+ if answer == "" {+ return+ }+ s.mu.Lock()+ defer s.mu.Unlock()+ for _, r := range s.calls {+ if r.answer != answer {+ continue+ }+ s.kb.Chunks = append([]map[string]any(nil), r.kb.Chunks...)+ s.kb.DocAggs = append([]map[string]any(nil), r.kb.DocAggs...)+ s.kb.Memory = append([]map[string]any(nil), r.kb.Memory...)+ s.kb.PreSummary = r.kb.PreSummary+ return+ }+}+// runDirect is the low/naive path: one hybrid search, no tool loop.//// It is the ONE Go implementation of Python orchestrator/direct.py::direct_searchdiff --git a/internal/rag/advanced_rag/agentic_rag_graph.go b/internal/rag/advanced_rag/agentic_rag_graph.goindex 5f117ae66a..fa6ae929ae 100644--- a/internal/rag/advanced_rag/agentic_rag_graph.go+++ b/internal/rag/advanced_rag/agentic_rag_graph.go@@ -50,6 +50,7 @@ import ("sync""time""unicode"+ "unicode/utf8""github.com/cespare/xxhash/v2""github.com/cloudwego/eino/compose"@@ -811,7 +812,9 @@ func fanoutLooksLikeQuery(line string, loose bool) bool {return false}}- if len(s) > fanoutMaxChars {+ // Python len() counts characters, not bytes; a byte-based cap would reject a+ // legitimate CJK fan-out well below the 160-character limit.+ if utf8.RuneCountInString(s) > fanoutMaxChars {return false}words := len(strings.Fields(s))@@ -821,6 +824,17 @@ func fanoutLooksLikeQuery(line string, loose bool) bool {return words <= fanoutMaxWords}+// fanoutLineBreak reports whether r is a Python str.splitlines() line boundary.+// Splitting on "\n" alone misses a lone "\r" and the other Unicode line breaks a+// model could emit, so the loose path would fuse several fan-out lines into one.+func fanoutLineBreak(r rune) bool {+ switch r {+ case '\n', '\r', '\v', '\f', 0x1c, 0x1d, 0x1e, 0x85, 0x2028, 0x2029:+ return true+ }+ return false+}+// parseFanouts mirrors Python _parse_fanouts: extract fan-outs from a// model reply, validating every entry's shape.func parseFanouts(text string) []string {@@ -837,11 +851,15 @@ func parseFanouts(text string) []string {// look like a search query are kept — answer sentences and source// lists are dropped.loose = true- for _, ln := range strings.Split(text, "\n") {- ln = strings.TrimSpace(strings.TrimLeft(ln, "-•0123456789. "))- if ln != "" {- raw = append(raw, ln)+ // Python `text.splitlines()`: split on every line boundary, not just "\n".+ for _, ln := range strings.FieldsFunc(text, fanoutLineBreak) {+ if strings.TrimSpace(ln) == "" {+ continue}+ // Python `ln.strip("-•0123456789. ").strip()` strips the bullet /+ // numbering cutset from BOTH ends, then trims whitespace, so a+ // trailing period/digit never leaks into the retrieval query.+ raw = append(raw, strings.TrimSpace(strings.Trim(ln, "-•0123456789. ")))}}kept := make([]string, 0, len(raw))@@ -1408,7 +1426,6 @@ func scaNode(ctx context.Context, deps RAGTools, st *AgenticState, logger *log.L// when the SCA produced no structured gaps.func unresolvedClueGaps(st *AgenticState) []orchestrator.MissingPiece {var gaps []orchestrator.MissingPiece- seen := map[string]bool{}for _, us := range st.UnresolvedSlots {clues, ok := us["question_clues"].([]string)if !ok {@@ -1419,10 +1436,9 @@ func unresolvedClueGaps(st *AgenticState) []orchestrator.MissingPiece {break}qc = strings.TrimSpace(qc)- if qc == "" || seen[qc] {+ if qc == "" {continue}- seen[qc] = truegaps = append(gaps, orchestrator.MissingPiece{What: qc, SearchHint: qc})}}@@ -1977,6 +1993,13 @@ func routeSCA(st *AgenticState, enableSCA bool, scaMaxRounds int) agenticNode {if st.NoProgress {return nodeFormalizeAnswer}+ // Evidence pool saturated at the SCA view cap: any further chunk lands+ // beyond what the SCA can read, so another search round cannot flip the+ // sufficiency verdict. Short-circuit straight to finalize.+ if st.KB != nil && len(st.KB.Chunks) >= SCAViewCap {+ _LOG.Printf("[SCA] evidence pool FULL (%d chunks >= SCA view cap %d); early-stopping to finalize_answer.", len(st.KB.Chunks), SCAViewCap)+ return nodeFormalizeAnswer+ }if !enableSCA {// medium: single research pass — the SCA verdict is informational only.return nodeFormalizeAnswer@@ -2224,7 +2247,7 @@ func RunSlotResearchPass(ctx context.Context, deps harness.SessionDeps, question}// Shared across sessions so duplicate retrievals are served from cache.- sharedToolCache := map[string]harness.ToolOutcome{}+ sharedToolCache := harness.NewToolCache()var sharedSearchQueries []stringsem := make(chan struct{}, slotSessionConcurrency)@@ -3032,15 +3055,10 @@ func NewAgenticLoop() AgenticLoop {// accumulates across the outer react loop's multiple rag() calls within a single// turn — which is exactly why Rag() must build deps.Cache before branching into// the outer loop (otherwise the "STOP calling rag again" guard would never fire).+// Those calls run concurrently, so the update itself is RAGCache.NoteUnanswerable+// (locked); this wrapper only keeps the call site's name.func recordConsecutiveUnanswerable(cache *RAGCache, verdict string) {- if cache == nil {- return- }- if verdict == VerdictSufficient {- cache.ConsecutiveUnanswerable = 0- } else {- cache.ConsecutiveUnanswerable++- }+ cache.NoteUnanswerable(verdict)}// graphRecursionLimit mirrors Python run_agentic_rag — the graph abortsdiff --git a/internal/rag/advanced_rag/agentic_rag_graph_test.go b/internal/rag/advanced_rag/agentic_rag_graph_test.goindex 1646037bd2..c07ac3e8a4 100644--- a/internal/rag/advanced_rag/agentic_rag_graph_test.go+++ b/internal/rag/advanced_rag/agentic_rag_graph_test.go@@ -167,6 +167,27 @@ func TestRouteSCADisabledSCAClosesOut(t *testing.T) {}}+// TestRouteSCAFullPoolShortCircuits pins the PR: a pool at/over the SCA view cap+// cannot flip the verdict, so routeSCA finalizes instead of researching — even+// when the verdict is insufficient and headroom remains.+func TestRouteSCAFullPoolShortCircuits(t *testing.T) {+ st := &AgenticState{+ Verdict: VerdictInsufficient,+ MaxLoops: 3,+ SearchRounds: 0,+ KB: &harness.Kbinfos{Chunks: make([]map[string]any, SCAViewCap)},+ }+ st.Deadline = time.Now().Add(120 * time.Second)+ if n := routeSCA(st, true, 3); n != nodeFormalizeAnswer {+ t.Fatalf("full pool: expected formalize, got %v", n)+ }+ // One chunk below the cap still takes the normal research path.+ st.KB.Chunks = make([]map[string]any, SCAViewCap-1)+ if n := routeSCA(st, true, 3); n != nodeQueryRewrite {+ t.Fatalf("below cap: expected rewrite, got %v", n)+ }+}+func TestRouteSCAInsufficientStartsRewrite(t *testing.T) {// Sufficient verdict closes out.st := &AgenticState{Verdict: VerdictSufficient, MaxLoops: 3}@@ -273,10 +294,12 @@ func TestExpandFanoutsFallbackOnBadJSON(t *testing.T) {mdl := &scriptedModel{}mdl.push("I cannot break this down.")got := ExpandFanouts(context.Background(), RAGTools{Model: mdl}, "single question")- // Bad JSON: the loop falls back to line-splitting the model's reply, which- // yields the reply itself (single line), not the raw question.- if len(got) != 1 || got[0] != "I cannot break this down." {- t.Fatalf("expected line-split fallback, got %v", got)+ // Bad JSON: the model answered in prose, so the loop line-splits the reply+ // (yielding the reply itself, not the raw question). The loose path strips the+ // Python `strip("-•0123456789. ")` cutset from both ends, so the trailing+ // period is dropped exactly as `_parse_fanouts` would.+ if len(got) != 1 || got[0] != "I cannot break this down" {+ t.Fatalf("expected line-split fallback without the trailing period, got %v", got)}}@@ -336,6 +359,51 @@ func TestFanoutLooksLikeQueryBounds(t *testing.T) {}}+// TestFanoutLooksLikeQueryCountsRunes pins Python len(): the 160-char cap counts+// code points, so a CJK fan-out at the limit must survive even though it exceeds+// 160 BYTES, and one over the limit must still be rejected.+func TestFanoutLooksLikeQueryCountsRunes(t *testing.T) {+ atLimit := strings.Repeat("中", fanoutMaxChars) // 160 chars / 480 bytes+ if !fanoutLooksLikeQuery(atLimit, false) {+ t.Errorf("a %d-character fan-out must pass: the cap counts characters, not bytes", fanoutMaxChars)+ }+ if fanoutLooksLikeQuery(atLimit+"中", false) {+ t.Errorf("a %d-character fan-out must be rejected", fanoutMaxChars+1)+ }+}++// TestParseFanoutsLooseStripsBothEnds pins Python `ln.strip("-•0123456789. ")`:+// the bullet/numbering cutset is stripped from BOTH ends, so a trailing period is+// not carried into the retrieval query (the leading "1." already was not).+func TestParseFanoutsLooseStripsBothEnds(t *testing.T) {+ got := parseFanouts("1. who opened the library.\n- when did it open?")+ want := []string{"who opened the library", "when did it open?"}+ if len(got) != len(want) {+ t.Fatalf("parseFanouts = %q, want %q", got, want)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Errorf("parseFanouts[%d] = %q, want %q", i, got[i], want[i])+ }+ }+}++// TestParseFanoutsSplitsOnAllLineBreaks pins Python `text.splitlines()`: a+// carry-return separated reply yields one fan-out per line, not a single fused+// blob (splitting on "\n" alone would keep the lone "\r" inline).+func TestParseFanoutsSplitsOnAllLineBreaks(t *testing.T) {+ got := parseFanouts("who opened it\rwhen did it open?")+ want := []string{"who opened it", "when did it open?"}+ if len(got) != len(want) {+ t.Fatalf("parseFanouts = %q, want %q", got, want)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Errorf("parseFanouts[%d] = %q, want %q", i, got[i], want[i])+ }+ }+}+// TestRenderSlotDraftMatchesPythonFormat pins Python _render_slot_draft// (agentic_rag_graph.py:1278-1326) exactly: the collected answer leads with its// evidence metadata, a resolved slot carries strength + terminal + evidence ids +@@ -610,7 +678,13 @@ type channelRetriever struct {}func (r *channelRetriever) Retrieve(_ context.Context, req harness.RetrieveRequest) ([]map[string]any, error) {- r.weights = append(r.weights, req.KeywordsSimilarityWeight)+ // Each leg names its weight explicitly; nil (the control was not supplied)+ // is recorded as -1 so the assertions below catch a leg that drops it.+ weight := -1.0+ if req.KeywordsSimilarityWeight != nil {+ weight = *req.KeywordsSimilarityWeight+ }+ r.weights = append(r.weights, weight)r.topN = append(r.topN, req.TopN)r.queries = append(r.queries, req.Query)return nil, nil@@ -1232,15 +1306,18 @@ func TestQueryRewriteFoldsUnresolvedCluesWhenNoGaps(t *testing.T) {func TestUnresolvedClueGapsCapsAtTwoClues(t *testing.T) {st := &AgenticState{UnresolvedSlots: []map[string]any{{"question_clues": []string{"a", "b", "c"}},- {"question_clues": []string{"a", "d"}}, // duplicate "a" is dropped+ {"question_clues": []string{"a", "d"}}, // Python appends without dedupe}}gaps := unresolvedClueGaps(st)- if len(gaps) != 3 {- t.Fatalf("gaps = %d, want 3 (2 from the first slot + 1 new from the second)", len(gaps))+ if len(gaps) != 4 {+ t.Fatalf("gaps = %d, want 4 (first two clues of each slot, duplicates kept)", len(gaps))}if gaps[0].What != "a" || gaps[0].SearchHint != "a" {t.Errorf("gaps[0] = %+v, want what=hint=\"a\"", gaps[0])}+ if gaps[2].What != "a" || gaps[3].What != "d" {+ t.Errorf("gaps = %+v, want the second slot's clues a,d appended verbatim", gaps)+ }}// TestBuildSCAClaimsFallsBackToEvidenceOnlyClaim pins Python :1072-1073: with no@@ -2490,25 +2567,25 @@ func TestRecordConsecutiveUnanswerableAcrossOuterRagCalls(t *testing.T) {// First outer rag() call — unsatisfying verdict bumps the counter to 1.recordConsecutiveUnanswerable(cache, VerdictInsufficient)- if cache.ConsecutiveUnanswerable != 1 {+ if cache.ConsecutiveUnanswerable() != 1 {t.Fatalf("after 1st outer rag() call: ConsecutiveUnanswerable = %d, want 1",- cache.ConsecutiveUnanswerable)+ cache.ConsecutiveUnanswerable())}// Second outer rag() call — still unsatisfying: counter must reach 2 so the// STOP guard can fire (the state the pre-fix code could never reach on the// outer path, because deps.Cache was nil and the increment was skipped).recordConsecutiveUnanswerable(cache, VerdictInsufficient)- if cache.ConsecutiveUnanswerable != 2 {+ if cache.ConsecutiveUnanswerable() != 2 {t.Fatalf("after 2nd outer rag() call: ConsecutiveUnanswerable = %d, want 2",- cache.ConsecutiveUnanswerable)+ cache.ConsecutiveUnanswerable())}// A satisfying verdict resets the streak, matching Python rag (:921-924).recordConsecutiveUnanswerable(cache, VerdictSufficient)- if cache.ConsecutiveUnanswerable != 0 {+ if cache.ConsecutiveUnanswerable() != 0 {t.Fatalf("after a SUFFICIENT verdict: ConsecutiveUnanswerable = %d, want 0",- cache.ConsecutiveUnanswerable)+ cache.ConsecutiveUnanswerable())}}diff --git a/internal/rag/advanced_rag/agentic_rag_test.go b/internal/rag/advanced_rag/agentic_rag_test.goindex 9fcfb409f5..8fb771d8ae 100644--- a/internal/rag/advanced_rag/agentic_rag_test.go+++ b/internal/rag/advanced_rag/agentic_rag_test.go@@ -19,7 +19,11 @@ package advanced_ragimport ("context""errors"+ "fmt"+ "io"+ "log""strings"+ "sync""testing""github.com/cloudwego/eino/schema"@@ -284,7 +288,7 @@ func TestResolveEffectiveQuestionHandlesEmpty(t *testing.T) {// rag() calls.func TestResearchStatusTrailerStopsAfterTwoUnanswerable(t *testing.T) {cache := NewRAGCache()- cache.ConsecutiveUnanswerable = 2+ cache.consecutiveUnanswerable = 2resp := &RunResponse{Verdict: VerdictInsufficient,SCAFeedback: "evidence is not yet sufficient",@@ -305,7 +309,7 @@ func TestResearchStatusTrailerStopsAfterTwoUnanswerable(t *testing.T) {// re-ask rather than tell the agent to stop.func TestResearchStatusTrailerInvitesFocusedReaskBelowTwo(t *testing.T) {cache := NewRAGCache()- cache.ConsecutiveUnanswerable = 1+ cache.consecutiveUnanswerable = 1resp := &RunResponse{Verdict: VerdictInsufficient,SCAFeedback: "evidence is not yet sufficient",@@ -329,7 +333,7 @@ func TestResearchStatusTrailerInvitesFocusedReaskBelowTwo(t *testing.T) {// missing SCA feedback all yield "".func TestResearchStatusTrailerSkipsSufficientOrEmpty(t *testing.T) {cache := NewRAGCache()- cache.ConsecutiveUnanswerable = 2+ cache.consecutiveUnanswerable = 2cases := []struct {name string@@ -501,3 +505,175 @@ func TestFitEvidenceTrimsOversizedEvidence(t *testing.T) {t.Fatal("evidence must not be trimmed to nothing")}}++// TestOuterReactSessionPublishMergesPerCallResults pins the locked merge: a+// round's rag calls run concurrently, so each publishes its own evidence and+// answer and none may be lost. Run with -race.+func TestOuterReactSessionPublishMergesPerCallResults(t *testing.T) {+ session := &outerReactSession{kb: &harness.Kbinfos{}, resp: &RunResponse{}}+ const calls = 4++ var wg sync.WaitGroup+ for i := 0; i < calls; i++ {+ wg.Add(1)+ go func(i int) {+ defer wg.Done()+ session.publish(+ &RunResponse{Answer: fmt.Sprintf("answer-%d", i), Verdict: fmt.Sprintf("verdict-%d", i)},+ &harness.Kbinfos{+ Chunks: []map[string]any{{"chunk_id": fmt.Sprintf("c%d", i)}},+ DocAggs: []map[string]any{{"doc_id": fmt.Sprintf("d%d", i)}},+ Memory: []map[string]any{{"id": fmt.Sprintf("m%d", i)}},+ })+ }(i)+ }+ wg.Wait()++ if len(session.kb.Chunks) != calls || len(session.kb.DocAggs) != calls || len(session.kb.Memory) != calls {+ t.Fatalf("merged evidence = %d chunks / %d doc_aggs / %d memory, want %d each",+ len(session.kb.Chunks), len(session.kb.DocAggs), len(session.kb.Memory), calls)+ }+ seen := map[string]bool{}+ for _, c := range session.kb.Chunks {+ id, _ := c["chunk_id"].(string)+ seen[id] = true+ }+ for i := 0; i < calls; i++ {+ if !seen[fmt.Sprintf("c%d", i)] {+ t.Errorf("merged chunks lost c%d: %v", i, seen)+ }+ }+ if !strings.HasPrefix(session.resp.Answer, "answer-") {+ t.Errorf("answer = %q, want one call's answer", session.resp.Answer)+ }+ if !strings.HasPrefix(session.resp.Verdict, "verdict-") {+ t.Errorf("verdict = %q, want one call's verdict", session.resp.Verdict)+ }+}++// TestOuterReactSessionToolCallKeepsSharedRequestIntact pins the per-call+// isolation: models.appendToolResults runs a round's rag calls concurrently+// (chat_tools.go), so a call must apply its rewritten question and its cleared+// images to a COPY — never to the request a concurrent call is reading.+// Run with -race.+func TestOuterReactSessionToolCallKeepsSharedRequestIntact(t *testing.T) {+ spec := harness.GetMode("naive")+ session := &outerReactSession{+ ctx: context.Background(),+ spec: spec,+ kb: &harness.Kbinfos{},+ resp: &RunResponse{Mode: spec},+ logger: log.New(io.Discard, "", 0),+ req: harness.RunRequest{+ Question: "original question",+ Images: []string{"data:image/png;base64,AAAA"},+ },+ }++ var wg sync.WaitGroup+ for _, q := range []string{"first question", "second question"} {+ wg.Add(1)+ go func(q string) {+ defer wg.Done()+ if _, err := session.ToolCall("rag", map[string]interface{}{"question": q}); err != nil {+ t.Errorf("ToolCall(%q): %v", q, err)+ }+ }(q)+ }+ wg.Wait()++ if session.req.Question != "original question" {+ t.Errorf("shared request question = %q, want it untouched (each call works on a copy)", session.req.Question)+ }+ if len(session.req.Images) != 1 {+ t.Errorf("shared request images = %v, want them untouched", session.req.Images)+ }+}++// ragTestChunkIDs lists a chunk pool's ids in order.+func ragTestChunkIDs(chunks []map[string]any) []string {+ out := make([]string, 0, len(chunks))+ for _, c := range chunks {+ id, _ := c["chunk_id"].(string)+ out = append(out, id)+ }+ return out+}++// TestRAGCacheConsecutiveUnanswerableIsSerialized pins that the counter lives+// behind the cache's lock: a round's concurrent rag() calls bump the SAME shared+// cache, so an unsynchronized increment would race and lose updates.+// Run with -race.+func TestRAGCacheConsecutiveUnanswerableIsSerialized(t *testing.T) {+ cache := NewRAGCache()+ const calls = 8++ var wg sync.WaitGroup+ for i := 0; i < calls; i++ {+ wg.Add(1)+ go func() {+ defer wg.Done()+ cache.NoteUnanswerable(VerdictInsufficient)+ }()+ }+ wg.Wait()++ if got := cache.ConsecutiveUnanswerable(); got != calls {+ t.Errorf("ConsecutiveUnanswerable = %d, want %d (increments must not be lost)", got, calls)+ }+ cache.NoteUnanswerable(VerdictSufficient)+ if got := cache.ConsecutiveUnanswerable(); got != 0 {+ t.Errorf("after a SUFFICIENT verdict = %d, want 0", got)+ }+ if got := (*RAGCache)(nil).ConsecutiveUnanswerable(); got != 0 {+ t.Errorf("nil cache = %d, want 0", got)+ }+ (*RAGCache)(nil).NoteUnanswerable(VerdictInsufficient) // must not panic+}++// TestOuterReactSessionSelectsWinningCallEvidence pins the citation fix: the+// terminal fold returns the LOWEST-INDEX terminal rag call's answer while+// publish() records completion order, so the session must hand back THAT call's+// evidence — otherwise the answer's [ID:n] markers address another call's chunks.+func TestOuterReactSessionSelectsWinningCallEvidence(t *testing.T) {+ session := &outerReactSession{kb: &harness.Kbinfos{}, resp: &RunResponse{}}+ winner := &harness.Kbinfos{+ Chunks: []map[string]any{{"chunk_id": "w0"}, {"chunk_id": "w1"}},+ DocAggs: []map[string]any{{"doc_id": "dw"}},+ Memory: []map[string]any{{"id": "mw"}},+ PreSummary: "winner summary",+ }+ loser := &harness.Kbinfos{Chunks: []map[string]any{{"chunk_id": "l0"}}}++ // The losing call finishes FIRST, i.e. completion order ≠ index order.+ session.publish(&RunResponse{Answer: "answer-loser"}, loser)+ session.publish(&RunResponse{Answer: "answer-winner"}, winner)++ if got := ragTestChunkIDs(session.kb.Chunks); len(got) != 3 || got[0] != "l0" {+ t.Fatalf("unselected union = %v, want the loser's chunk first (that is the misalignment)", got)+ }++ session.selectEvidence("answer-winner")++ if got := ragTestChunkIDs(session.kb.Chunks); len(got) != 2 || got[0] != "w0" || got[1] != "w1" {+ t.Errorf("chunks after selection = %v, want the winner's [w0 w1] in its own order", got)+ }+ if len(session.kb.DocAggs) != 1 || len(session.kb.Memory) != 1 || session.kb.PreSummary != "winner summary" {+ t.Errorf("winner's evidence fields not restored: %+v", session.kb)+ }+}++// TestOuterReactSessionSelectEvidenceKeepsUnionWithoutAMatch keeps the fallback:+// an answer no rag call produced (the terminal tool was summarize_document, or+// the outer model answered itself) leaves the union in place.+func TestOuterReactSessionSelectEvidenceKeepsUnionWithoutAMatch(t *testing.T) {+ session := &outerReactSession{kb: &harness.Kbinfos{}, resp: &RunResponse{}}+ session.publish(&RunResponse{Answer: "answer-a"}, &harness.Kbinfos{Chunks: []map[string]any{{"chunk_id": "a0"}}})++ session.selectEvidence("a summarize_document result")+ session.selectEvidence("")++ if got := ragTestChunkIDs(session.kb.Chunks); len(got) != 1 || got[0] != "a0" {+ t.Errorf("chunks = %v, want the union kept when no call matches", got)+ }+}diff --git a/internal/rag/advanced_rag/harness/action_session.go b/internal/rag/advanced_rag/harness/action_session.goindex 75260cd517..dfac24ed50 100644--- a/internal/rag/advanced_rag/harness/action_session.go+++ b/internal/rag/advanced_rag/harness/action_session.go@@ -925,10 +925,11 @@ var (Type: "function",Function: ToolFunction{Name: "retrieve",- Description: ("Keyword-first search of the fixed document corpus. Pass natural-" +- "language queries; returns SHORT snippets of the most relevant " +- "passages (exact-term matched where possible). Use multiple queries " +- "to cover different aspects. Supports 1-3 queries per call."),+ Description: `WHEN TO CALL: Use when you know or suspect exact surface terms or keywords in the corpus (names, titles, codes, phrases). Best as the first recall pass; send 1-3 queries covering different facets.` ++ `DO NOT CALL: When you already hold a doc_id and need to read it (use list_chunks); when the answer shares no surface words with any query (use search_chunks); for counting or enumerating a whole document.` ++ `ARGUMENTS: query — array of 1-3 strings (natural-language queries). Note: doc_scope exists inside the executor but is NOT a declared parameter; do not pass it.` ++ `OUTPUT: Short exact-term-matched snippets, each carrying its doc_id and chunk id. Status ok means new evidence entered the pool; redundant means everything was already there.` ++ `IF IT FAILS: miss (empty payload) means this query matched nothing — rephrase or switch to search_chunks; do not conclude the corpus lacks the fact. redundant means stop re-searching and emit a state patch.`,Parameters: arrayParam("", 1, 3),},}@@ -937,10 +938,11 @@ var (Type: "function",Function: ToolFunction{Name: "list_chunks",- Description: ("Deep-read the FULL text of one document by doc_id (returned in " +- "retrieve snippets). Use for enumeration / count / arithmetic answers " +- "when snippets are insufficient. Returns all chunks of the document " +- "in reading order. One doc_id per call."),+ Description: `WHEN TO CALL: You need the FULL text of one document (enumeration, counts, arithmetic over many passages) and you already have its doc_id from a prior tool result.` ++ `DO NOT CALL: When you only need a single passage (use search_chunks or retrieve first); when you have no doc_id yet (locate it via navigate_tree or search_chunks first).` ++ `ARGUMENTS: doc_id — string, the document id seen in a retrieve / search_chunks / navigate result. ONLY doc_id is accepted; there is no chunk_ids argument, and the tool returns the whole document (capped at 30 chunks).` ++ `OUTPUT: All chunks of the document in reading order. ok = new evidence; redundant = already in pool.` ++ `IF IT FAILS: An unknown or blank doc_id yields an empty result (query-level miss, not a dataset fact) — pick a different doc_id or locate one first. Do not treat this as a reason to disable the tool.`,Parameters: map[string]any{"type": "object","properties": map[string]any{@@ -958,18 +960,11 @@ var (Type: "function",Function: ToolFunction{Name: "search_chunks",- Description: ("SEMANTIC retrieval (hybrid vector+BM25) with COMPILED-STRUCTURE " +- "EXPANSION. Use as the PRIMARY recall tool when exact-term " +- "``retrieve`` returns nothing useful, or when the dataset is large and " +- "you are unsure which document holds the answer — the answer passage " +- "may share NO surface words with the query. " +- "Compiled expansion: automatically appends related chunks from the " +- "dataset's compiled structure (page index, tree/heading hierarchy, " +- "knowledge graph, wiki pages when present) so a semantic hit carries " +- "its structural neighbours (parent/child headings, sibling pages). " +- "If the dataset has NO compiled structure (incl. no wiki), expansion " +- "is a no-op — no error, just semantic hits. " +- "Returns snippet chunks ranked by relevance. 1-2 queries per call."),+ Description: `WHEN TO CALL: Primary semantic recall. Use when exact retrieve returns nothing useful, when the corpus is large and you are unsure which document holds the answer, or when the answer passage shares no surface words with your query. Send 1-2 queries.` ++ `DO NOT CALL: When you already have a doc_id and want to read that document (use list_chunks); when a single exact passage would be found faster by grep-style retrieve.` ++ `ARGUMENTS: query — array of 1-2 strings. Compiled-structure expansion is automatic and a no-op on datasets without compiled structure, so no extra argument is needed.` ++ `OUTPUT: Relevance-ranked snippet chunks, possibly with structural neighbours (parent/child headings, sibling pages) appended. ok = new evidence; redundant = already seen.` ++ `IF IT FAILS: miss means this query matched nothing — change the angle or fall back to retrieve or navigate_tree. Re-issuing a near-duplicate query is skipped as redundant, so vary the query instead of paraphrasing it.`,Parameters: arrayParam("", 1, 2),},}@@ -978,11 +973,11 @@ var (Type: "function",Function: ToolFunction{Name: "web_search",- Description: ("Search the open WEB. Use ONLY when the needed fact is world " +- "knowledge / recent event / not covered by the fixed corpus — e.g. " +- "a current event, a person's alive-now status, or a statistic newer " +- "than the corpus. If the fact plausibly lives in the documents, " +- "prefer corpus tools (retrieve/search_chunks) first. 1-2 queries per call."),+ Description: `WHEN TO CALL: The needed fact is world knowledge, a recent event, or newer than the corpus (a current event, a person's alive-now status, a fresh statistic). This tool only appears when a web provider is configured.` ++ `DO NOT CALL: When the fact plausibly lives in the fixed corpus — prefer retrieve or search_chunks first. For corpus-only questions this tool is unavailable.` ++ `ARGUMENTS: query — array of 1-2 strings.` ++ `OUTPUT: Web results shaped like corpus chunks, merged into the same evidence pool.` ++ `IF IT FAILS: error (no provider) — it will not appear at all this session; if it does appear and fails, switch to corpus tools permanently and do not retry it.`,Parameters: arrayParam("", 1, 2),},}@@ -997,23 +992,11 @@ var (Type: "function",Function: ToolFunction{Name: "navigate_tree",- Description: ("LOCATE the RIGHT DOCUMENT among MANY before deep-reading. Use it " +- "BEFORE search_chunks when the dataset is large and you have no " +- "doc_id yet — it routes by TOPIC/CLUSTERING similarity over the " +- "compiled document-navigation tree (not exact surface words), so it " +- "finds the document even when your query words differ from its text. " +- "Returns candidate doc_ids + a first-chunk summary of each. " +- "This is the FIRST hop of a navigation chain: " +- "navigate_tree(query) -> doc_id -> navigate_structure(doc_id, ...) " +- "-> list_chunks(doc_id, chunk_ids). " +- "Use when: the question names a topic/entity/alias but you do not " +- "know which document discusses it; search_chunks returned scattered " +- "hits across many docs and you must pick the source. " +- "Do NOT use if you already hold a doc_id (go straight to " +- "navigate_structure) or if the answer is likely a single exact " +- "passage (prefer retrieve/search_chunks). " +- "If the dataset has no compiled document navigation tree, it returns " +- "empty — fall back to search_chunks."),+ Description: `WHEN TO CALL: The question names a topic, entity, or alias but you do NOT know which document discusses it, especially on a large corpus. Routes by topic or cluster similarity over the compiled navigation tree.` ++ `DO NOT CALL: When you already hold a doc_id (go straight to navigate_structure); when the answer is likely a single exact passage (use retrieve or search_chunks).` ++ `ARGUMENTS: query — string, the topic / entity / alias whose document(s) to locate. Note: keywords is read by the executor but is NOT a declared parameter; do not pass it.` ++ `OUTPUT: Candidate doc_ids plus a first-chunk summary of each; these become your known-docs set for the next step.` ++ `IF IT FAILS: empty (no_structure) means the dataset has no compiled navigation tree — immediately switch to search_chunks. A second such empty disables this tool for the rest of the session, so do not retry it.`,Parameters: map[string]any{"type": "object","properties": map[string]any{@@ -1031,18 +1014,11 @@ var (Type: "function",Function: ToolFunction{Name: "navigate_structure",- Description: ("PINPOINT A PASSAGE inside ONE document using its compiled structure " +- "(heading/catalog tree, concept mindmap, or entity graph) — the " +- "in-document counterpart of navigate_tree. " +- "Use AFTER you know the doc_id (from navigate_tree / search_chunks / " +- "retrieve) and need to find where the answer lives WITHOUT reading " +- "every chunk. Returns the structure outline annotated with matching " +- "chunk_ids (reading-order aware). Then call list_chunks(doc_id, " +- "chunk_ids) to read exactly those. " +- "kind: 'catalog' (default) for page-index/heading/timeline trees, " +- "'mindmap' for concept maps, 'graph' for entity-relation graphs. " +- "If the document has NO compiled structure, an empty <doc/> is " +- "returned — fall back to list_chunks to read the full document."),+ Description: `WHEN TO CALL: You know the doc_id and need to PINPOINT where the answer lives inside that one document, without reading every chunk. The in-document counterpart of navigate_tree.` ++ `DO NOT CALL: When you have no doc_id yet; when the document has no compiled structure (use list_chunks to read the full document).` ++ `ARGUMENTS: doc_id — string, required. query — string, what to locate within the document. kind — enum catalog / mindmap / graph, default catalog (compiled-structure kind).` ++ `OUTPUT: The structure outline annotated with matching chunk_ids, reading-order aware. ok = useful hits; poor (chunk_ptrs = 0) means it drilled to nothing usable.` ++ `IF IT FAILS: empty (no_structure) — try another doc_id or kind, or fall back to list_chunks / search_chunks. poor — read the full document via list_chunks(doc_id). A second empty disables the tool for the session.`,Parameters: map[string]any{"type": "object","properties": map[string]any{@@ -1059,19 +1035,11 @@ var (Type: "function",Function: ToolFunction{Name: "calculate",- Description: ("COMPUTE a numeric answer by generating and safely running code. " +- "MANDATORY whenever the question asks you to DERIVE a number by " +- "combining facts you found (sum/difference/percentage/ratio/sort/" +- "compare/difference in length/age, price, area, growth, etc.) — do " +- "NOT do arithmetic mentally. Language-neutral: the question and " +- "facts may be in ANY language (English, Chinese, ...); pass the " +- "numbers verbatim as written in the evidence regardless of language. " +- "Steps: (1) collect every needed number first (retrieve / " +- "search_chunks / navigate_* / list_chunks); (2) call calculate with " +- "the question + ALL those numbers; (3) report the computed result " +- "verbatim. If a needed number is missing, search for it first — do " +- "not estimate. If the answer IS one of the stated numbers (no " +- "combination needed), answer directly without this tool."),+ Description: `WHEN TO CALL: The question asks you to DERIVE a number by combining facts you found (sum / difference / percentage / ratio / sort / compare / length / age / price / area / growth). NEVER do arithmetic mentally.` ++ `DO NOT CALL: When the answer IS one of the stated numbers (no combination needed) — answer directly. When a needed number is still missing — retrieve it first; do not estimate.` ++ `ARGUMENTS: question — string, the user question verbatim. facts — array of strings, the numbers or facts found in evidence, verbatim (keep the original language; pass them exactly as written).` ++ `OUTPUT: an object with expression and result — report the computed result verbatim.` ++ `IF IT FAILS: poor (no numeric answer derivable) — retrieve more numbers, or answer directly if the answer is already stated. Never fabricate a computation.`,Parameters: map[string]any{"type": "object","properties": map[string]any{@@ -1177,6 +1145,10 @@ type Toolset struct {// DisabledTools holds tools proven unavailable this session (no compiled// structure of their kind).DisabledTools map[string]bool+ // mu guards DisabledTools: one Toolset is shared by a session's concurrent+ // rag calls (models.appendToolResults runs a round's tool calls in parallel),+ // so the map must not be written and read unsynchronized.+ mu sync.Mutex// Exec runs the tools.Exec ToolExecutor}@@ -1209,7 +1181,7 @@ func (t *Toolset) ActiveToolSpecs() []ToolSpec {if name == "web_search" && !t.HasWebSearch {continue}- if t.DisabledTools[name] {+ if t.IsDisabled(name) {continue}if s, ok := ToolMap[name]; ok {@@ -1226,6 +1198,8 @@ func (t *Toolset) DisableTool(name string) {if _, ok := ToolMap[name]; !ok {return}+ t.mu.Lock()+ defer t.mu.Unlock()if t.DisabledTools == nil {t.DisabledTools = map[string]bool{}}@@ -1234,6 +1208,8 @@ func (t *Toolset) DisableTool(name string) {// IsDisabled reports whether name has been disabled this session.func (t *Toolset) IsDisabled(name string) bool {+ t.mu.Lock()+ defer t.mu.Unlock()return t.DisabledTools[name]}@@ -1665,8 +1641,10 @@ type SessionState struct {// ToolChars is the running total of tool-payload chars emitted so far// (O(1) accounting; Python tracks the same counter as _tool_chars).ToolChars int- // ToolCache avoids re-executing an identical (name, args) call.- ToolCache map[string]ToolOutcome+ // ToolCache avoids re-executing an identical (name, args) call. It is the+ // per-round cache shared with the round's other sessions, so it is a guarded+ // *ToolCache rather than a bare map.+ ToolCache *ToolCache// SearchQueries accumulates retrieval queries for near-dup detection.SearchQueries []string// SkippedDup counts near-duplicate retrievals suppressed so far.@@ -1821,7 +1799,7 @@ func (s *SessionState) toolNode(ctx context.Context) error {}used := s.ToolCharsif s.ToolCache == nil {- s.ToolCache = map[string]ToolOutcome{}+ s.ToolCache = NewToolCache()}seenQueries := append([]string(nil), s.SearchQueries...)skipped := 0@@ -1867,14 +1845,16 @@ func (s *SessionState) toolNode(ctx context.Context) error {continue}- // Same-session cache: avoid re-running an identical (name, args) call.+ // Same-round cache: avoid re-running an identical (name, args) call.// The cache only avoids RE-EXECUTING; a response is still returned for// every declared call, because the assistant message declared it.+ // The cache is shared by a round's concurrent sessions, hence the guarded+ // Get/Put.cacheKey := callCacheKey(c)- oc, cached := s.ToolCache[cacheKey]+ oc, cached := s.ToolCache.Get(cacheKey)if !cached {oc = ExecuteTool(ctx, s.Tools, c.Name, c.Args)- s.ToolCache[cacheKey] = oc+ s.ToolCache.Put(cacheKey, oc)}if q != "" {seenQueries = append(seenQueries, q)@@ -2387,6 +2367,46 @@ func callCacheKey(c ToolCall) string {return c.Name + "|" + string(raw)}+// ToolCache is the per-round tool-outcome cache. One instance is shared by a+// round's CONCURRENT sessions: Python builds a single dict per round+// (agentic_rag_graph.py:1407) and hands it to every session (:2030), and only+// asyncio's cooperative scheduling keeps that safe. Go's sessions are goroutines+// and concurrent map access is a FATAL error there, so the map is guarded.+type ToolCache struct {+ mu sync.Mutex+ m map[string]ToolOutcome+}++// NewToolCache returns an empty, concurrency-safe tool cache.+func NewToolCache() *ToolCache { return &ToolCache{m: map[string]ToolOutcome{}} }++// Get returns the cached outcome for key. A nil cache never holds anything, so a+// session constructed without one simply re-executes (the pre-existing behavior).+func (c *ToolCache) Get(key string) (ToolOutcome, bool) {+ if c == nil {+ return ToolOutcome{}, false+ }+ c.mu.Lock()+ defer c.mu.Unlock()+ oc, ok := c.m[key]+ return oc, ok+}++// Put stores the outcome for key. Two identical calls that lose the race both+// compute and both store — Python's dict behaves the same way under asyncio (the+// cache avoids re-executing DELIBERATE repeats, not simultaneous ones).+func (c *ToolCache) Put(key string, oc ToolOutcome) {+ if c == nil {+ return+ }+ c.mu.Lock()+ defer c.mu.Unlock()+ if c.m == nil {+ c.m = map[string]ToolOutcome{}+ }+ c.m[key] = oc+}+// stripUnpairedToolCalls removes assistant tool_calls that never received a// tool response — a malformed history makes providers reject the next request// ("tool call result does not follow tool call").@@ -3040,7 +3060,7 @@ type SessionDeps struct {// Returns an empty Result (no states) when no model is configured or the// session fails — mirroring Python, which logs and returns// Result(messages=[], new_states=[]) rather than propagating.-func RunActionSession(ctx context.Context, deps SessionDeps, direction string, parent State, deadlineLeft float64, baseSummary string, sharedToolCache map[string]ToolOutcome, sharedSearchQueries []string) Result {+func RunActionSession(ctx context.Context, deps SessionDeps, direction string, parent State, deadlineLeft float64, baseSummary string, sharedToolCache *ToolCache, sharedSearchQueries []string) Result {system := loadPrompt(deps.Prompts, "action_run")seedUser := fmt.Sprintf("Direction: %s\n\nState:\n%s", direction, parent.RenderSlots())@@ -3091,7 +3111,7 @@ func RunActionSession(ctx context.Context, deps SessionDeps, direction string, pDirection: direction,}if st.ToolCache == nil {- st.ToolCache = map[string]ToolOutcome{}+ st.ToolCache = NewToolCache()}// The graph loop is bounded by the turn budget and the deadline; the contextdiff --git a/internal/rag/advanced_rag/harness/action_session_test.go b/internal/rag/advanced_rag/harness/action_session_test.goindex f30a707a02..8658f5cb4d 100644--- a/internal/rag/advanced_rag/harness/action_session_test.go+++ b/internal/rag/advanced_rag/harness/action_session_test.go@@ -8,6 +8,7 @@ import ("strings""sync""testing"+ "unicode/utf8""github.com/cloudwego/eino/compose""github.com/cloudwego/eino/schema"@@ -95,7 +96,7 @@ func TestToolNodeContinuesLadderInCode(t *testing.T) {DeadlineLeft: 60,Direction: "who created Culdcept",NavRuleID: llmRung,- ToolCache: map[string]ToolOutcome{},+ ToolCache: NewToolCache(),PendingCalls: []ToolCall{{ID: "call-1", Name: "navigate_structure"},},@@ -152,7 +153,7 @@ func TestToolNodeLeavesLadderAloneWithNoPendingRule(t *testing.T) {DeadlineLeft: 60,Direction: "who created Culdcept",NavRuleID: "", // ladder finished- ToolCache: map[string]ToolOutcome{},+ ToolCache: NewToolCache(),PendingCalls: []ToolCall{{ID: "call-1", Name: "retrieve"},},@@ -1089,3 +1090,101 @@ func TestRenderPromptUsesLoader(t *testing.T) {// ---------------------------------------------------------------------------// Helpers// ---------------------------------------------------------------------------++// ---------------------------------------------------------------------------+// Tool spec / schema contract (moved here from action_session_schema_test.go)+// ---------------------------------------------------------------------------++// playbookAnchors are the 5-section contract every tool description must carry+// (mirrors PLAYBOOK_ANCHORS in the Python action-session schema test).+var playbookAnchors = []string{"WHEN TO CALL", "DO NOT CALL", "ARGUMENTS", "OUTPUT", "IF IT FAILS"}++// executorSupportedParams lists the params the executor actually consumes per+// tool (mirrors _EXECUTOR_SUPPORTED). The schema MUST NOT declare any param+// outside this set — otherwise the model is told to fill an argument the runtime+// silently ignores (the list_chunks(chunk_ids) ghost bug). Note: doc_scope /+// keywords are supported by the executor but intentionally NOT declared in the+// schema (keep the description honest with the implementation).+var executorSupportedParams = map[string]map[string]bool{+ "retrieve": {"query": true, "doc_scope": true},+ "search_chunks": {"query": true},+ "list_chunks": {"doc_id": true},+ "navigate_tree": {"query": true, "keywords": true},+ "navigate_structure": {"doc_id": true, "query": true, "kind": true},+ "calculate": {"question": true, "facts": true},+ "web_search": {"query": true},+}++// TestToolSpecsHavePlaybookSections pins the PR: each of the 7 tools documents+// the 5-section contract, within the token budget (cap 1200 chars).+func TestToolSpecsHavePlaybookSections(t *testing.T) {+ for _, name := range allTools {+ desc := ToolMap[name].Function.Description+ for _, anchor := range playbookAnchors {+ if !strings.Contains(desc, anchor) {+ t.Errorf("%s description missing anchor %q", name, anchor)+ }+ }+ if n := utf8.RuneCountInString(desc); n > 1200 {+ t.Errorf("%s description too long: %d chars (cap 1200)", name, n)+ }+ }+}++// TestActiveToolSpecsToolSurface pins mode -> exposed tool count: low=0,+// medium/high=7, ultra=8, web-hidden=6.+func TestActiveToolSpecsToolSurface(t *testing.T) {+ cases := []struct {+ mode string+ web bool+ want int+ }{+ {"low", true, 0},+ {"medium", true, 7},+ {"high", true, 7},+ {"ultra", true, 8},+ {"medium", false, 6},+ }+ for _, c := range cases {+ ts := &Toolset{ThinkingMode: c.mode, HasWebSearch: c.web}+ if got := len(ts.ActiveToolSpecs()); got != c.want {+ t.Errorf("mode=%s web=%t: tools = %d, want %d", c.mode, c.web, got, c.want)+ }+ }+}++// TestSchemaParamsMatchExecutor pins that no schema declares a param the+// executor cannot consume (no ghost args).+func TestSchemaParamsMatchExecutor(t *testing.T) {+ for _, name := range allTools {+ props, _ := ToolMap[name].Function.Parameters["properties"].(map[string]any)+ supported := executorSupportedParams[name]+ for param := range props {+ if param == "decision" {+ continue+ }+ if !supported[param] {+ t.Errorf("%s declares unsupported param %q", name, param)+ }+ }+ }+}++// TestActionRunPromptHasPlaybook pins that action_run.md exposes the TOOL+// PLAYBOOK and only references real tools.+func TestActionRunPromptHasPlaybook(t *testing.T) {+ prompt := loadPrompt(nil, "action_run")+ if !strings.Contains(prompt, "TOOL PLAYBOOK") {+ t.Error("action_run prompt missing TOOL PLAYBOOK")+ }+ for _, tool := range []string{"navigate_structure", "calculate", "graph_explore"} {+ if !strings.Contains(prompt, tool) {+ t.Errorf("action_run prompt missing tool %q", tool)+ }+ }+ for _, name := range append(append([]string{}, allTools...), GraphExploreTool) {+ if _, ok := ToolMap[name]; !ok {+ t.Errorf("%q referenced by the playbook is not in ToolMap", name)+ }+ }+}diff --git a/internal/rag/advanced_rag/harness/arithmetic.go b/internal/rag/advanced_rag/harness/arithmetic.goindex 8939c7d082..d77cf02f93 100644--- a/internal/rag/advanced_rag/harness/arithmetic.go+++ b/internal/rag/advanced_rag/harness/arithmetic.go@@ -1380,11 +1380,14 @@ func dateDiff(a, b any) (int64, error) {if err != nil {return 0, err}- days := db.Sub(da).Hours() / 24- if days < 0 {- days = -days- }- return int64(days), nil+ // Both dates are midnight UTC, so a second-based difference is exact and+ // avoids time.Duration's ~292-year int64 nanosecond ceiling, which SATURATES+ // (every longer span came back as 106751 days).+ seconds := db.Unix() - da.Unix()+ if seconds < 0 {+ seconds = -seconds+ }+ return seconds / 86400, nil}func parseISODate(s string) (time.Time, error) {diff --git a/internal/rag/advanced_rag/harness/arithmetic_test.go b/internal/rag/advanced_rag/harness/arithmetic_test.goindex 81a98bc30e..0bc9bca35b 100644--- a/internal/rag/advanced_rag/harness/arithmetic_test.go+++ b/internal/rag/advanced_rag/harness/arithmetic_test.go@@ -563,3 +563,27 @@ func TestComputeFromFactsFitsToContextBudget(t *testing.T) {t.Errorf("value = %q, want 44", cf.Value)}}++// TestDateDiffCountsCalendarDays pins that date_diff counts calendar days rather+// than a time.Duration: the latter saturates at its ~292-year int64 nanosecond+// ceiling, so every longer span came back as 106751 days.+func TestDateDiffCountsCalendarDays(t *testing.T) {+ cases := []struct {+ a, b string+ want int64+ }{+ {"1941-07-28", "1959-07-17", 6563}, // the prompt's own example+ {"1607-05-14", "2020-01-01", 150712}, // 412 years: past the Duration ceiling+ {"2020-01-01", "1607-05-14", 150712}, // order-independent (abs)+ {"1607-05-14", "1607-05-14", 0},+ }+ for _, tc := range cases {+ got, err := dateDiff(tc.a, tc.b)+ if err != nil {+ t.Fatalf("dateDiff(%q, %q): %v", tc.a, tc.b, err)+ }+ if got != tc.want {+ t.Errorf("dateDiff(%q, %q) = %d, want %d", tc.a, tc.b, got, tc.want)+ }+ }+}diff --git a/internal/rag/advanced_rag/harness/doc_fetch.go b/internal/rag/advanced_rag/harness/doc_fetch.goindex 7d4878f7dd..10dbb206d7 100644--- a/internal/rag/advanced_rag/harness/doc_fetch.go+++ b/internal/rag/advanced_rag/harness/doc_fetch.go@@ -130,20 +130,49 @@ func summarizeDocument(ctx context.Context, deps SearchDeps, docID string, maxTo}// Python :942-948 — the document becomes part of the evidence set, so its- // [ID]s stay citable; only the blocks added here are returned.- startIdx := 0- if deps.KB != nil {- startIdx = len(deps.KB.Chunks)- deps.KB.Merge(chunks, aggs)- } else {+ // [ID]s stay citable; only the blocks of the chunks read here are returned.+ //+ // The blocks are picked by the SOURCE CHUNK, not by the pre-merge chunk+ // count: KBPrompt skips a chunk with no content and stops when the token+ // budget is exhausted, so block N is not chunk N — a slice taken at the+ // pre-merge count can drop readable blocks, or point past the end and return+ // nil for a document that was read fine (Python's `blocks[start_idx:]` has the+ // same flaw; Go hits it more often because Merge deduplicates a document+ // chunk that is already pooled, so the pre-merge count can even equal the+ // post-merge count). Merge reports the pool position of every chunk this+ // fetch contributed — deduplicated ones included — which makes the mapping+ // exact; iterating it keeps the document's reading order.+ if deps.KB == nil {deps.KB = &Kbinfos{}- deps.KB.Merge(chunks, aggs)}- blocks := prompts.KBPrompt(deps.KB.Chunks, budget)- if startIdx >= len(blocks) {+ added := deps.KB.Merge(chunks, aggs)+ blocks, sources := prompts.KBPromptWithSourceIndices(deps.KB.Chunks, budget)+ blockAt := make(map[int]string, len(sources))+ for i, src := range sources {+ if _, seen := blockAt[src]; !seen {+ blockAt[src] = blocks[i]+ }+ }+ // One block per POOL POSITION. `added` is positional per OCCURRENCE: a chunk+ // the reader served twice (offset paging over an unstable order) or a chunk+ // already pooled is reported at the same position again, so appending per+ // occurrence would hand the model the same block — and its tokens — twice.+ fresh := make([]string, 0, len(added))+ seenPositions := make(map[int]struct{}, len(added))+ for _, pos := range added {+ if _, seen := seenPositions[pos]; seen {+ continue+ }+ seenPositions[pos] = struct{}{}+ if block, ok := blockAt[pos]; ok {+ fresh = append(fresh, block)+ }+ }+ // Nothing of this document survived the budget: the pool is already at the+ // model window, so there is no block to hand back.+ if len(fresh) == 0 {return nil}- fresh := blocks[startIdx:]// Python :952-959 — without do_refer the model is told not to cite, so the// rules must not be handed to it.diff --git a/internal/rag/advanced_rag/harness/doc_fetch_test.go b/internal/rag/advanced_rag/harness/doc_fetch_test.gonew file mode 100644index 0000000000..9382ee6813--- /dev/null+++ b/internal/rag/advanced_rag/harness/doc_fetch_test.go@@ -0,0 +1,127 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "strings"+ "testing"+)++// docChunkStub serves one document's chunks in reading order, paged.+type docChunkStub struct {+ chunks []map[string]any+}++func (d *docChunkStub) DocChunks(_ context.Context, req DocChunksRequest) ([]map[string]any, error) {+ if req.Offset >= len(d.chunks) {+ return nil, nil+ }+ end := req.Offset + req.Limit+ if end > len(d.chunks) {+ end = len(d.chunks)+ }+ return d.chunks[req.Offset:end], nil+}++func docFetchDeps(pool, docChunks []map[string]any) SearchDeps {+ return SearchDeps{+ KbIDs: []string{"kb-1"},+ TenantID: "tenant-1",+ KB: &Kbinfos{Chunks: pool},+ DocChunks: &docChunkStub{chunks: docChunks},+ }+}++// TestSummarizeDocumentReturnsBlocksForAlreadyPooledChunks covers the case the+// pre-merge chunk count gets wrong on its own: Merge deduplicates a chunk that is+// already pooled, so the pool does not grow and a slice taken at the pre-merge+// count starts past the end of the rendered blocks — the tool then reported that+// a document it had just read had no readable chunks.+func TestSummarizeDocumentReturnsBlocksForAlreadyPooledChunks(t *testing.T) {+ pool := []map[string]any{{"chunk_id": "c1", "content": "Pooled body."}}+ docChunks := []map[string]any{{"chunk_id": "c1", "content": "Pooled body."}}++ got := summarizeDocument(context.Background(), docFetchDeps(pool, docChunks), "doc-1", 100000)++ if len(got) != 1 {+ t.Fatalf("blocks = %d (%v), want the document's block even though it was already pooled", len(got), got)+ }+ if !strings.Contains(got[0], "Pooled body.") {+ t.Errorf("block = %q, want the document's content", got[0])+ }+}++// TestSummarizeDocumentSelectsBlocksBySourceChunk covers a skipped chunk: KBPrompt+// renders no block for a chunk without content, so every later block is one+// position earlier than its chunk. Slicing by the chunk count dropped the first+// document block.+func TestSummarizeDocumentSelectsBlocksBySourceChunk(t *testing.T) {+ pool := []map[string]any{+ {"chunk_id": "empty", "content": ""}, // renders no block+ }+ docChunks := []map[string]any{+ {"chunk_id": "c1", "content": "First document body."},+ {"chunk_id": "c2", "content": "Second document body."},+ }++ got := summarizeDocument(context.Background(), docFetchDeps(pool, docChunks), "doc-1", 100000)++ if len(got) != 2 {+ t.Fatalf("blocks = %d (%v), want both document chunks", len(got), got)+ }+ if !strings.Contains(got[0], "First document body.") || !strings.Contains(got[1], "Second document body.") {+ t.Errorf("blocks = %v, want the document's chunks in reading order", got)+ }+}++// TestSummarizeDocumentReturnsNothingWhenNoChunkRenders keeps the guard: when the+// fetched chunks contribute no renderable block there is nothing to hand back,+// and in particular no unrelated pooled block.+func TestSummarizeDocumentReturnsNothingWhenNoChunkRenders(t *testing.T) {+ pool := []map[string]any{{"chunk_id": "p0", "content": "Pooled evidence."}}+ docChunks := []map[string]any{{"chunk_id": "c1", "content": " "}}++ got := summarizeDocument(context.Background(), docFetchDeps(pool, docChunks), "doc-1", 100000)++ if len(got) != 0 {+ t.Fatalf("blocks = %v, want nothing for a document with no renderable chunk", got)+ }+}++// TestSummarizeDocumentDeduplicatesRepeatedSourcePositions covers the paging+// hazard: offset paging over an unstable order can serve the same chunk on two+// pages, and Merge reports a position per OCCURRENCE, not per distinct chunk. The+// block of that one pooled chunk was therefore appended twice — paying its tokens+// into the prompt twice, on top of a budget that was already spent.+func TestSummarizeDocumentDeduplicatesRepeatedSourcePositions(t *testing.T) {+ pool := []map[string]any{{"chunk_id": "p0", "content": "Pooled evidence."}}+ docChunks := []map[string]any{+ {"chunk_id": "c1", "content": "Document body."},+ {"chunk_id": "c1", "content": "Document body."}, // same chunk served twice+ {"chunk_id": "c2", "content": "Second body."},+ }++ got := summarizeDocument(context.Background(), docFetchDeps(pool, docChunks), "doc-1", 100000)++ if len(got) != 2 {+ t.Fatalf("blocks = %d (%v), want one block per distinct fetched chunk", len(got), got)+ }+ if !strings.Contains(got[0], "Document body.") || !strings.Contains(got[1], "Second body.") {+ t.Errorf("blocks = %v, want the document's chunks in reading order, each once", got)+ }+}diff --git a/internal/rag/advanced_rag/harness/kbinfos.go b/internal/rag/advanced_rag/harness/kbinfos.goindex 548b56d0de..605e37545c 100644--- a/internal/rag/advanced_rag/harness/kbinfos.go+++ b/internal/rag/advanced_rag/harness/kbinfos.go@@ -77,7 +77,15 @@ type SearchParams struct {type SearchFn func(ctx context.Context, p SearchParams) ([]map[string]any, []map[string]any)// Kbinfos is the shared accumulation store.+//+// ONE Kbinfos is shared by a round's CONCURRENT sessions (SessionDeps.KB; see+// RunSlotResearchPass), so its mutable state is guarded by mu. Python needs no+// lock — its sessions are coroutines and every admission stretch is await-free+// (action_session.py:691-700), so asyncio cannot interleave two sessions'+// batches. Go runs the same sessions on goroutines, so the critical sections+// Python gets for free have to be locked explicitly: see Admit.type Kbinfos struct {+ mu sync.MutexChunks []map[string]anyDocAggs []map[string]any// Memory is the lossless store of raw retrieved chunks backing the (lossy)@@ -96,10 +104,99 @@ type Kbinfos struct {// HasChunks mirrors Python `bool(kbinfos.get("chunks"))`.func (k *Kbinfos) HasChunks() bool { return len(k.Chunks) > 0 }+// Admit runs fn as ONE critical section over the evidence pool.+//+// The granularity is the caller's, and it must span the stretch Python leaves+// await-free: ONE query's candidate batch, because the awaits sit in the OUTER+// query loop (action_session.py:691-700). Locking per chunk would let two+// sessions' batches interleave into pool orders Python cannot produce, and the+// pool order is observable — it drives the rendered prompt order and the `ID: n`+// numbering, extractRelevantEvidence's first-4 pick, and which chunks make it in+// under the cap.+//+// Inside fn do, per chunk, exactly what _admit_evidence does, in its order:+//+// p.Full() → skip the chunk. The cap check precedes the per-call dedup+// (:638-646), so a rejected chunk is NOT marked seen and is+// retried once room frees.+// then the call-local dedup (`seen`) and payload/ids bookkeeping;+// p.Add(c) → dedup against the live pool and append.+func (k *Kbinfos) Admit(fn func(p *PoolAdmitter)) {+ if k == nil {+ // No pool bound: the callback still runs — a tool hands its passages back+ // either way — but nothing is pooled and the cap never blocks, which is+ // what the callers' old `if deps.KB != nil` guard did.+ fn(&PoolAdmitter{})+ return+ }+ k.mu.Lock()+ defer k.mu.Unlock()+ fn(&PoolAdmitter{k: k})+}++// PoolAdmitter is the pool handle, valid only inside Kbinfos.Admit's callback.+type PoolAdmitter struct{ k *Kbinfos }++// Full mirrors the early stop at the top of _admit_evidence: at the cap the+// chunk is skipped, and nothing about it is recorded (so a rejected chunk is+// retried once room frees).+//+// The "pool FULL" line belongs here for the same reason it lives inside+// _admit_evidence (:639-645), and it is emitted once per PROCESS — Python+// documents _EVIDENCE_POOL_STATE as a "Per-process flag" (:62-64). sync.Once+// because concurrent sessions call this under their own pool lock.+func (p *PoolAdmitter) Full() bool {+ if p.k == nil || len(p.k.Chunks) < evidencePoolCap {+ return false+ }+ evidencePoolFullLogged.Do(func() {+ _LOG.Printf("[Action Session] evidence pool FULL (%d chunks >= cap %d); early-stopping admit of further chunks.", len(p.k.Chunks), evidencePoolCap)+ })+ return true+}++// evidencePoolFullLogged mirrors Python _EVIDENCE_POOL_STATE["full_logged"]: a+// PER-PROCESS flag (action_session.py:62-64), so the line is emitted once per+// process — not once per rejected chunk, and NOT once per session. The pool never+// shrinks mid-process, so no reset is needed.+var evidencePoolFullLogged sync.Once++// Add appends c unless the LIVE pool already holds it, reporting whether it+// appended — Python's "new to the shared pool" return.+//+// The test is against the live pool, not a snapshot taken when the call started+// (Python's `kb_seen`, :690). That snapshot is exact only because nothing can+// interleave with it; under Go's parallelism it would re-append a chunk another+// session just pooled, which Python can never do.+//+// There is deliberately NO cap check here: the compiled-expansion path appends+// uncapped in Python too, which is what pushes the pool past _EVIDENCE_POOL_CAP+// (see the comment on evidencePoolCap). Callers that admit user-facing search+// hits check Full() first.+func (p *PoolAdmitter) Add(c map[string]any) bool {+ if p.k == nil {+ return false+ }+ if p.Index(c) >= 0 {+ return false+ }+ p.k.Chunks = append(p.k.Chunks, c)+ return true+}++// Index is the pool position of c's identity, or -1 when the pool lacks it (and+// for an unbound pool): the positional contract of Merge — a deduped chunk still+// reports its position.+func (p *PoolAdmitter) Index(c map[string]any) int {+ if p.k == nil {+ return -1+ }+ return indexOfChunk(p.k.Chunks, chunkKey(c))+}+// Merge appends the given chunks/aggs, deduplicating by chunkKey, and returns the// GLOBAL index (position in k.Chunks) of every contributed chunk — deduped ones-// included. len(result) is therefore NOT the count of new evidence: callers that-// need that compare len(k.Chunks) before and after.+// included.//// Mirrors orchestrator/direct.py:_merge_kbinfos, including its early return on an// empty chunk list (neither chunks nor doc_aggs are merged).@@ -107,20 +204,16 @@ func (k *Kbinfos) Merge(chunks, aggs []map[string]any) []int {if len(chunks) == 0 {return nil}- seen := make(map[string]bool, len(k.Chunks))- for _, c := range k.Chunks {- seen[chunkKey(c)] = true- }- var added []int- for _, c := range chunks {- kk := chunkKey(c)- if !seen[kk] {- seen[kk] = true- k.Chunks = append(k.Chunks, c)+ added := make([]int, 0, len(chunks))+ // One critical section for the whole merge: the dedup is against the live+ // pool, so a concurrent session cannot slip the same chunk in between.+ k.Admit(func(p *PoolAdmitter) {+ for _, c := range chunks {+ p.Add(c)+ // Deduped chunks get an entry too: the return value is positional.+ added = append(added, p.Index(c))}- // Deduped chunks get an entry too: the return value is positional.- added = append(added, indexOfChunk(k.Chunks, kk))- }+ })k.MergeDocAggs(aggs)return added}@@ -132,9 +225,11 @@ func (k *Kbinfos) Merge(chunks, aggs []map[string]any) []int {// the search tool must record a search's aggregations even when every chunk it// returned was already pooled.func (k *Kbinfos) MergeDocAggs(aggs []map[string]any) {- if len(aggs) == 0 {+ if k == nil || len(aggs) == 0 {return}+ k.mu.Lock()+ defer k.mu.Unlock()dseen := make(map[string]bool, len(k.DocAggs)+len(aggs))for _, d := range k.DocAggs {dseen[docAggKey(d)] = true@@ -191,6 +286,12 @@ func chunkText(c map[string]any) string {// present; otherwise falls back to a content-derived key so chunks without an// id still dedup correctly (and never collide on a shared empty key).//+// The content branch reuses chunkText — the SAME alias chain the rest of the+// harness reads (content_with_weight -> content -> text). Reading only the first+// two made two id-less chunks that carry just "text" fall through to the+// doc-level fallback and share one key, so Merge/MemoryAdd discarded distinct+// evidence.+//// This diverges from Python _chunk_key (`chunk_id or id or id(ck)`): Python's// final fallback is `id(ck)` — the dict object's memory address — which means// two equivalent chunks returned as distinct objects never share a key, so@@ -203,12 +304,7 @@ func chunkKey(c map[string]any) string {if id, ok := c["id"].(string); ok && id != "" {return "id:" + id}- content := ""- if t, ok := c["content_with_weight"].(string); ok {- content = t- } else if t, ok := c["content"].(string); ok {- content = t- }+ content := chunkText(c)if content == "" {// No stable identity: fall back to the doc reference so at least// per-document grouping is preserved (rarely reached).diff --git a/internal/rag/advanced_rag/harness/kbinfos_test.go b/internal/rag/advanced_rag/harness/kbinfos_test.goindex 2dc7209e5e..2ad44d9b70 100644--- a/internal/rag/advanced_rag/harness/kbinfos_test.go+++ b/internal/rag/advanced_rag/harness/kbinfos_test.go@@ -1,6 +1,10 @@package harness-import "testing"+import (+ "fmt"+ "sync"+ "testing"+)// TestMergeSkipsAllWhenNoChunks mirrors Python _merge_kbinfos's early return:// `if not result or not result.get("chunks"): return`. When the incoming chunk@@ -49,3 +53,151 @@ func TestMergeDedupsSameDocID(t *testing.T) {t.Fatalf("same-doc_id aggs should dedup, got %d", len(kb.DocAggs))}}++// TestChunkKeyUsesTextFallback pins that the dedup key reads the SAME alias chain+// as chunkText (content_with_weight -> content -> text). Reading only the first+// two made two id-less, text-only chunks from one document share the doc-level+// fallback key, so Merge/MemoryAdd discarded distinct evidence.+func TestChunkKeyUsesTextFallback(t *testing.T) {+ a := map[string]any{"text": "alpha", "doc_id": "d1", "docnm_kwd": "doc"}+ b := map[string]any{"text": "beta", "doc_id": "d1", "docnm_kwd": "doc"}+ if chunkKey(a) == chunkKey(b) {+ t.Fatalf("distinct text-only chunks must not share a key: %q", chunkKey(a))+ }+ // The same text under any alias must produce the same key.+ if c := map[string]any{"content": "alpha"}; chunkKey(a) != chunkKey(c) {+ t.Errorf("text and content aliases must agree: %q vs %q", chunkKey(a), chunkKey(c))+ }+ // A chunk with no text at all still lands on the doc-level fallback.+ if got := chunkKey(map[string]any{"doc_id": "d1", "docnm_kwd": "doc"}); got == "" {+ t.Error("doc-level fallback key must not be empty")+ }+}++// TestMergeKeepsDistinctTextOnlyChunks pins the end-to-end consequence for the+// evidence pool: two id-less chunks that share only a document must both survive.+func TestMergeKeepsDistinctTextOnlyChunks(t *testing.T) {+ kb := &Kbinfos{}+ added := kb.Merge([]map[string]any{+ {"text": "alpha", "doc_id": "d1", "docnm_kwd": "doc"},+ {"text": "beta", "doc_id": "d1", "docnm_kwd": "doc"},+ }, nil)+ if len(kb.Chunks) != 2 {+ t.Fatalf("chunks = %d, want 2 (distinct text-only evidence must not collapse)", len(kb.Chunks))+ }+ if len(added) != 2 {+ t.Fatalf("added = %v, want 2 global indices", added)+ }+ // Re-merging identical content still dedups.+ kb.Merge([]map[string]any{{"text": "alpha", "doc_id": "d1", "docnm_kwd": "doc"}}, nil)+ if len(kb.Chunks) != 2 {+ t.Fatalf("identical text must still dedup, chunks = %d", len(kb.Chunks))+ }+}++// TestMemoryAddKeepsDistinctTextOnlyChunks pins the same consequence for the+// lossless memory store.+func TestMemoryAddKeepsDistinctTextOnlyChunks(t *testing.T) {+ kb := &Kbinfos{}+ MemoryAdd(kb, []map[string]any{+ {"text": "alpha", "doc_id": "d1", "docnm_kwd": "doc"},+ {"text": "beta", "doc_id": "d1", "docnm_kwd": "doc"},+ })+ if len(kb.Memory) != 2 {+ t.Fatalf("memory = %d, want 2 (distinct text-only evidence must not collapse)", len(kb.Memory))+ }+ MemoryAdd(kb, []map[string]any{{"text": "alpha", "doc_id": "d1", "docnm_kwd": "doc"}})+ if len(kb.Memory) != 2 {+ t.Fatalf("identical text must still dedup, memory = %d", len(kb.Memory))+ }+}++// TestKbinfosAdmitIsAtomic pins the pool's invariants under the concurrency the+// pool actually sees: one round's sessions run at the same time and share ONE+// Kbinfos (SessionDeps.KB; RunSlotResearchPass starts them as goroutines).+//+// Python gets both invariants for free — asyncio never preempts the await-free+// admit stretch (_admit_evidence is a plain def, action_session.py:615, and the+// per-query loop has no await, :691-700) — so Go has to lock the same stretch.+//+// The pool starts one chunk short of the cap, and both sessions offer the SAME+// first chunk. Whichever batch wins the lock pools it — once — and the cap then+// stops every further chunk, so the assertions hold whatever the schedule does:+// they fail only when the batches interleave (which is what an unlocked+// check-then-append does: both sessions read 59 and both append).+func TestKbinfosAdmitIsAtomic(t *testing.T) {+ pool := &Kbinfos{}+ for i := 0; i < evidencePoolCap-1; i++ {+ pool.Admit(func(p *PoolAdmitter) {+ p.Add(map[string]any{"chunk_id": fmt.Sprintf("p%d", i)})+ })+ }+ if len(pool.Chunks) != evidencePoolCap-1 {+ t.Fatalf("fixture pool = %d chunks, want %d", len(pool.Chunks), evidencePoolCap-1)+ }++ var wg sync.WaitGroup+ for _, tag := range []string{"a", "b"} {+ wg.Add(1)+ go func(tag string) {+ defer wg.Done()+ pool.Admit(func(p *PoolAdmitter) {+ for _, c := range []map[string]any{+ {"chunk_id": "shared"},+ {"chunk_id": tag},+ } {+ if p.Full() {+ continue+ }+ p.Add(c)+ }+ })+ }(tag)+ }+ wg.Wait()++ if len(pool.Chunks) != evidencePoolCap {+ t.Errorf("pool = %d chunks, want exactly the cap %d: a serialized batch stops at the cap, an interleaved one overshoots",+ len(pool.Chunks), evidencePoolCap)+ }+ counts := map[string]int{}+ for _, c := range pool.Chunks {+ id, _ := c["chunk_id"].(string)+ counts[id]+++ }+ if counts["shared"] != 1 {+ t.Errorf("chunk %q pooled %d time(s), want exactly 1: the pool must be deduped against its LIVE contents", "shared", counts["shared"])+ }+ if counts["a"]+counts["b"] != 0 {+ t.Errorf("session-private chunks were pooled (%d a, %d b) although only one slot was free", counts["a"], counts["b"])+ }+}++// TestToolCacheIsConcurrencySafe exercises the cache the way a round does: the+// SAME instance handed to concurrent sessions (Python builds one dict per round,+// agentic_rag_graph.py:1407, and passes it to every session, action_session.py:2030).+// Run under -race this is what a bare map cannot survive — concurrent map writes+// are a fatal error in Go, not merely a race report.+func TestToolCacheIsConcurrencySafe(t *testing.T) {+ cache := NewToolCache()+ var wg sync.WaitGroup+ for i := 0; i < 16; i++ {+ wg.Add(1)+ go func(i int) {+ defer wg.Done()+ key := fmt.Sprintf("tool-%d", i%4) // deliberate key collisions+ if _, ok := cache.Get(key); !ok {+ cache.Put(key, ToolOutcome{Reason: key})+ }+ }(i)+ }+ wg.Wait()++ for i := 0; i < 4; i++ {+ key := fmt.Sprintf("tool-%d", i)+ oc, ok := cache.Get(key)+ if !ok || oc.Reason != key {+ t.Errorf("cache[%q] = (%+v, %v), want the stored outcome", key, oc, ok)+ }+ }+}diff --git a/internal/rag/advanced_rag/harness/memory.go b/internal/rag/advanced_rag/harness/memory.goindex b738af5778..aab01ba53b 100644--- a/internal/rag/advanced_rag/harness/memory.go+++ b/internal/rag/advanced_rag/harness/memory.go@@ -34,8 +34,10 @@ import (// The store lives on Kbinfos.Memory so it travels with the request.const (- // grepMaxChunks caps how many chunks one grep returns (Python- // memory._GREP_MAX_CHUNKS = 6).+ // grepMaxChunks is Python memory.grep's default cap (_GREP_MAX_CHUNKS = 6).+ // MemoryGrep takes the limit explicitly and does NOT substitute this for a+ // non-positive value (Python has no such guard), so callers that want the+ // Python default pass it.grepMaxChunks = 6// grepMaxSentences caps sentences kept per chunk (hit + context).grepMaxSentences = 4@@ -91,6 +93,11 @@ func MemoryAdd(kb *Kbinfos, chunks []map[string]any) {if kb == nil || len(chunks) == 0 {return}+ // One critical section: memory.add is an await-free stretch in Python, so+ // asyncio can never interleave two sessions' adds into it (the pool is shared+ // across a round's concurrent sessions — see Kbinfos.Admit).+ kb.mu.Lock()+ defer kb.mu.Unlock()seen := make(map[string]struct{}, len(kb.Memory))for _, c := range kb.Memory {seen[chunkKey(c)] = struct{}{}@@ -124,6 +131,8 @@ func MemorySize(kb *Kbinfos) int {// MemoryClear mirrors Python memory.clear.func MemoryClear(kb *Kbinfos) {if kb != nil {+ kb.mu.Lock()+ defer kb.mu.Unlock()kb.Memory = nil}}@@ -134,13 +143,15 @@ func MemoryClear(kb *Kbinfos) {// terms are plain strings (entities / numbers / key phrases) as emitted by the// analysis LLM. Each returned chunk carries a narrowed "content" so the caller// can splice it straight into an evidence list. Empty on no-hit / no-memory.+//+// limit is the maximum number of chunks returned and is NOT normalized: Python's+// grep has no such guard, so a limit <= 0 makes the `len(hits) >= limit` check+// fire on the first hit (at most one chunk comes back). Callers wanting Python's+// default pass grepMaxChunks.func MemoryGrep(kb *Kbinfos, terms []string, limit int) []map[string]any {if kb == nil || len(kb.Memory) == 0 || len(terms) == 0 {return nil}- if limit <= 0 {- limit = grepMaxChunks- }patterns, prefixPatterns := compileTerms(terms)if len(patterns) == 0 && len(prefixPatterns) == 0 {return nil@@ -173,6 +184,13 @@ func MemoryGrep(kb *Kbinfos, terms []string, limit int) []map[string]any {"doc_id": c["doc_id"],"chunk_id": c["chunk_id"],})+ // This branch continues past the shared cap below, so enforce the+ // limit here too — otherwise a memory store full of matching short+ // chunks comes back whole. Python's memory.grep enforces it in the+ // same place.+ if len(hits) >= limit {+ break+ }}continue}diff --git a/internal/rag/advanced_rag/harness/memory_test.go b/internal/rag/advanced_rag/harness/memory_test.goindex 971df79758..e1ad0693e7 100644--- a/internal/rag/advanced_rag/harness/memory_test.go+++ b/internal/rag/advanced_rag/harness/memory_test.go@@ -112,3 +112,35 @@ func TestChunkTextPrefersWeighted(t *testing.T) {t.Errorf("chunkText = %q, want %q", got, "primary only")}}++// TestMemoryGrepLimitAppliesToShortChunks pins the returned-chunk cap: the+// short-chunk branch continues past the shared check, so it must enforce the+// limit itself — otherwise a memory store full of matching short chunks comes+// back whole instead of at most limit entries.+func TestMemoryGrepLimitAppliesToShortChunks(t *testing.T) {+ mem := make([]map[string]any, 0, 10)+ for i := 0; i < 10; i++ {+ suffix := string(rune('a' + i)) // short chunk: well under shortChunkChars+ mem = append(mem, memChunk("rifampicin note "+suffix, "d1", "c"+suffix))+ }+ kb := &Kbinfos{Memory: mem}++ hits := MemoryGrep(kb, []string{"rifampicin"}, 2)+ if len(hits) != 2 {+ t.Fatalf("hits = %d, want the limit 2 (a short chunk must not bypass the cap)", len(hits))+ }+ if hits[0]["content"] != "rifampicin note a" {+ t.Errorf("hit content = %v, want the whole short chunk", hits[0]["content"])+ }++ // limit <= 0 is NOT normalized to the default: Python's grep has no such+ // guard, so the `len(hits) >= limit` check fires on the first hit.+ if got := len(MemoryGrep(kb, []string{"rifampicin"}, 0)); got != 1 {+ t.Errorf("limit=0 hits = %d, want 1 (Python grep does not normalize limit)", got)+ }++ // Python's default (6) is passed explicitly and caps at 6.+ if got := len(MemoryGrep(kb, []string{"rifampicin"}, grepMaxChunks)); got != grepMaxChunks {+ t.Errorf("default-limit hits = %d, want %d", got, grepMaxChunks)+ }+}diff --git a/internal/rag/advanced_rag/harness/orchestrator/sufficient_context.go b/internal/rag/advanced_rag/harness/orchestrator/sufficient_context.goindex 33cbd456c5..c6a97627c7 100644--- a/internal/rag/advanced_rag/harness/orchestrator/sufficient_context.go+++ b/internal/rag/advanced_rag/harness/orchestrator/sufficient_context.go@@ -21,6 +21,7 @@ import ("fmt""regexp""strings"+ "unicode/utf8""ragflow/internal/rag/advanced_rag/harness""ragflow/internal/rag/prompts"@@ -145,7 +146,7 @@ func SufficientContextAgent(ctx context.Context, deps SCADeps, question string,"overall_draft": overallDraft,})_LOG.Printf("[SCA] unified review of %s (reports only, %d chars; overall draft %d chars)",- fmtClaimCount(len(claims)), len(claimsContext), len(overallDraft))+ fmtClaimCount(len(claims)), utf8.RuneCountInString(claimsContext), utf8.RuneCountInString(overallDraft))result, err := deps.Model.GenJSON(ctx, prompt)if err != nil {@@ -166,7 +167,7 @@ func SufficientContextAgent(ctx context.Context, deps SCADeps, question string,if !ok {continue}- cid := strings.TrimSpace(fmt.Sprint(m["claim_id"]))+ cid := strOf(m["claim_id"])if cid == "" {continue}@@ -176,12 +177,12 @@ func SufficientContextAgent(ctx context.Context, deps SCADeps, question string,if s := strings.TrimSpace(fmt.Sprint(firstNonEmpty(um["assertion"], um["reason"]))); s != "" {ungrounded = append(ungrounded, s)}- } else if s := strings.TrimSpace(fmt.Sprint(u)); s != "" {+ } else if s := strOf(u); s != "" {ungrounded = append(ungrounded, s)}}claimsOut[cid] = ClaimVerdict{- Grounded: truthy(m["grounded"]),+ Grounded: coerceBool(m["grounded"]),Ungrounded: ungrounded,MissingInformation: parseMissingInformation(m["missing_information"]),}@@ -195,19 +196,19 @@ func SufficientContextAgent(ctx context.Context, deps SCADeps, question string,if !ok {continue}- sqText := strings.TrimSpace(fmt.Sprint(m["sub_query"]))+ sqText := strOf(m["sub_query"])if sqText == "" {continue}- entry := SubQuery{SubQuery: sqText, Satisfied: truthy(m["satisfied"])}+ entry := SubQuery{SubQuery: sqText, Satisfied: coerceBool(m["satisfied"])}if !entry.Satisfied {- entry.MissingFact = strings.TrimSpace(fmt.Sprint(m["missing_fact"]))- entry.SearchHint = strings.TrimSpace(fmt.Sprint(m["search_hint"]))+ entry.MissingFact = strOf(m["missing_fact"])+ entry.SearchHint = strOf(m["search_hint"])}subQueries = append(subQueries, entry)}- isSufficient := truthy(data["is_sufficient"])+ isSufficient := coerceBool(data["is_sufficient"])// Failsafe: the SCA judged the context insufficient but returned an EMPTY// claims array (a known degradation on very long prompts). Without a gap, the@@ -237,7 +238,7 @@ func SufficientContextAgent(ctx context.Context, deps SCADeps, question string,IsSufficient: isSufficient,Confidence: clamp(data["confidence"]),Contradictions: stringList(data["contradictions"]),- Reasoning: strings.TrimSpace(fmt.Sprint(data["reasoning"])),+ Reasoning: strOf(data["reasoning"]),SubQueries: subQueries,Claims: claimsOut,}@@ -315,7 +316,6 @@ func renderClaimContext(claims []ClaimDraft, kb *harness.Kbinfos) string {continue}block := fmt.Sprintf("Claim %s (draft):\n%s", c.ID, c.Draft)- used += len(block) + 2var anchors []stringfor _, eid := range c.EvidenceIDs {ck, ok := id2chunk[strings.TrimSpace(eid)]@@ -337,11 +337,23 @@ func renderClaimContext(claims []ClaimDraft, kb *harness.Kbinfos) string {}}if len(anchors) > 0 {- anchorText := strings.Join(anchors, " | ")- block += "\n Evidence: " + anchorText- used += len(anchorText) + 4+ block += "\n Evidence: " + strings.Join(anchors, " | ")+ }+ // Apply the budget to the COMPLETE block before appending it, trimming it+ // to what remains: accounting the block only AFTER appending let a single+ // oversized claim blow straight past scaClaimsContextMax (the cap the+ // comment claims bounds the whole rendered context). Cut by rune+ // (Python's str[:N]) so the trim never splits a multibyte character.+ remaining := scaClaimsContextMax - used+ if remaining <= 0 {+ break+ }+ if r := []rune(block); len(r) > remaining {+ block = string(r[:remaining])}blocks = append(blocks, block)+ // The block plus the "\n\n" join separator.+ used += utf8.RuneCountInString(block) + 2if used >= scaClaimsContextMax {break}@@ -370,8 +382,11 @@ func renderOverallDraft(claims []ClaimDraft) string {return "(no overall draft)"}draft := strings.Join(parts, "\n")- if len(draft) > scaClaimsContextMax {- draft = draft[:scaClaimsContextMax]+ // Truncate by RUNE, not byte: Python slices str[:N] by code point, and a byte+ // slice through a multibyte (e.g. CJK) rune would emit invalid UTF-8 (or+ // replacement characters) into the SCA prompt.+ if r := []rune(draft); len(r) > scaClaimsContextMax {+ draft = string(r[:scaClaimsContextMax])}return draft}@@ -505,19 +520,26 @@ func clamp(v any) float64 {return f}-// truthy mirrors Python's builtin bool() on the raw JSON value, exactly as the-// Python sufficient_context path evaluates these fields (e.g.-// bool(item.get("grounded")), bool(result.get("is_sufficient"))). Python's bool()-// treats ANY non-empty string as truthy — including the literal "false" or "0" —-// because they are non-empty. We must not string-parse booleans here, or a model-// that returns grounded:"false" as a string would wrongly collapse to False while-// Python keeps it True. Only an empty string / 0 / nil / False is falsy.-func truthy(v any) bool {+// coerceBool reads an LLM-reported boolean, tolerating the string serialisations+// a drifted model reply may carry. It deliberately does NOT use raw truthiness:+// a non-empty string is truthy in both Go and Python, so bool("false") and+// bool("0") are TRUE — which would silently INVERT a verdict, marking+// insufficient context sufficient or an ungrounded claim grounded.+//+// The recognised true/false spellings are mapped explicitly; an empty or+// unrecognised value fails CLOSED (false), matching this codebase's convention+// that an unusable verdict is INSUFFICIENT (see the SCA-unavailable path in+// agentic_rag_graph.go, which treats a timeout/unparsable reply as insufficient).+func coerceBool(v any) bool {switch t := v.(type) {case bool:return tcase string:- return t != ""+ switch strings.ToLower(strings.TrimSpace(t)) {+ case "true", "1", "yes", "y", "on":+ return true+ }+ return falsecase float64:return t != 0case int:@@ -527,19 +549,29 @@ func truthy(v any) bool {case nil:return false}- return true+ return false+}++// strOf renders a raw JSON field as a trimmed string, mapping an absent or null+// value (nil) to "". fmt.Sprint(nil) yields the literal "<nil>", which would+// otherwise masquerade as a real claim id, sub-query, gap or reasoning text.+func strOf(v any) string {+ if v == nil {+ return ""+ }+ return strings.TrimSpace(fmt.Sprint(v))}func parseMissingInformation(v any) []MissingPiece {var out []MissingPiecefor _, m := range asSlice(v) {if mm, ok := m.(map[string]any); ok {- what := strings.TrimSpace(fmt.Sprint(mm["what"]))- hint := strings.TrimSpace(fmt.Sprint(mm["search_hint"]))+ what := strOf(mm["what"])+ hint := strOf(mm["search_hint"])if what != "" || hint != "" {out = append(out, MissingPiece{What: what, SearchHint: hint})}- } else if s := strings.TrimSpace(fmt.Sprint(m)); s != "" {+ } else if s := strOf(m); s != "" {out = append(out, MissingPiece{What: s})}}diff --git a/internal/rag/advanced_rag/harness/orchestrator/sufficient_context_test.go b/internal/rag/advanced_rag/harness/orchestrator/sufficient_context_test.goindex 90593a6942..34b2a927b1 100644--- a/internal/rag/advanced_rag/harness/orchestrator/sufficient_context_test.go+++ b/internal/rag/advanced_rag/harness/orchestrator/sufficient_context_test.go@@ -249,13 +249,14 @@ func TestSCABoostAdaptsVerdict(t *testing.T) {// Query rewriter// ----------------------------------------------------------------------------func TestClampAndTruthy(t *testing.T) {+func TestClampCoercesNumericDrift(t *testing.T) {if got := clamp(float64(1.5)); got != 1.0 {t.Errorf("clamp(1.5) = %v, want 1.0", got)}if got := clamp(float64(-0.2)); got != 0.0 {t.Errorf("clamp(-0.2) = %v, want 0.0", got)}+ // Numeric drift: a string number is still parsed.if got := clamp("0.4"); got != 0.4 {t.Errorf("clamp(\"0.4\") = %v, want 0.4", got)}@@ -263,14 +264,164 @@ func TestClampAndTruthy(t *testing.T) {if got := clamp("not a number"); got != 1.0 {t.Errorf("clamp(bad) = %v, want 1.0", got)}- for _, v := range []any{true, "true", "TRUE", "1", "yes", "false", "0", float64(1)} {- if !truthy(v) {- t.Errorf("truthy(%v) = false, want true", v)+}++// TestCoerceBoolParsesStringDrift pins that a boolean serialised as a string is+// read by MEANING, not by raw truthiness: "false"/"0" are false, not true. An+// empty or unrecognised value fails CLOSED (false), matching the "unusable+// verdict is INSUFFICIENT" convention.+func TestCoerceBoolParsesStringDrift(t *testing.T) {+ for _, v := range []any{true, "true", "TRUE", " True ", "1", "yes", "Y", "on", float64(1)} {+ if !coerceBool(v) {+ t.Errorf("coerceBool(%#v) = false, want true", v)}}- for _, v := range []any{false, "", float64(0), nil} {- if truthy(v) {- t.Errorf("truthy(%v) = true, want false", v)+ for _, v := range []any{false, "false", "FALSE", " false ", "0", "no", "N", "off", "", float64(0), nil, "maybe"} {+ if coerceBool(v) {+ t.Errorf("coerceBool(%#v) = true, want false", v)}}}++// TestSCAParsesStringBooleans pins the drift fix end to end: a model that+// serialises its verdict booleans as strings must not have them inverted.+func TestSCAParsesStringBooleans(t *testing.T) {+ mdl := &stubJSONModel{value: map[string]any{+ "is_sufficient": "false",+ "sub_queries": []any{+ map[string]any{"sub_query": "revenue", "satisfied": "false",+ "missing_fact": "2023 revenue", "search_hint": "annual report"},+ },+ "claims": []any{+ map[string]any{"claim_id": "c1", "grounded": "false"},+ },+ }}+ got := SufficientContextAgent(context.Background(), SCADeps{+ KB: &harness.Kbinfos{},+ Model: mdl,+ }, "q", []ClaimDraft{{ID: "c1", Draft: "d"}})+ if got.IsSufficient {+ t.Error(`is_sufficient:"false" must read as false, not true`)+ }+ if len(got.SubQueries) != 1 || got.SubQueries[0].Satisfied {+ t.Errorf("sub_queries = %+v, want one unsatisfied", got.SubQueries)+ }+ if cv := got.Claims["c1"]; cv.Grounded {+ t.Error(`grounded:"false" must read as false, not true`)+ }++ // The "true" spelling still parses.+ mdl = &stubJSONModel{value: map[string]any{"is_sufficient": "true"}}+ got = SufficientContextAgent(context.Background(), SCADeps{+ KB: &harness.Kbinfos{},+ Model: mdl,+ }, "q", []ClaimDraft{{ID: "c1", Draft: "d"}})+ if !got.IsSufficient {+ t.Error(`is_sufficient:"true" must read as true`)+ }+}++// TestSCAIgnoresAbsentFields pins that an absent (nil) field never renders as the+// literal "<nil>": a claim without claim_id and a sub-query without sub_query are+// rejected, and the optional text fields stay empty instead of becoming "<nil>".+func TestSCAIgnoresAbsentFields(t *testing.T) {+ mdl := &stubJSONModel{value: map[string]any{+ "sub_queries": []any{+ map[string]any{"satisfied": false}, // no sub_query+ map[string]any{"sub_query": "ok", "satisfied": false}, // no missing_fact/search_hint+ },+ "claims": []any{+ map[string]any{"grounded": true}, // no claim_id+ map[string]any{"claim_id": "c1", "grounded": true}, // no missing_information+ },+ }}+ got := SufficientContextAgent(context.Background(), SCADeps{+ KB: &harness.Kbinfos{},+ Model: mdl,+ }, "q", []ClaimDraft{{ID: "c1", Draft: "d"}})++ if _, ok := got.Claims["<nil>"]; ok {+ t.Error(`a claim with no claim_id must be rejected, not keyed as "<nil>"`)+ }+ if _, ok := got.Claims["c1"]; !ok {+ t.Error("the claim with a real claim_id must be kept")+ }+ if len(got.SubQueries) != 1 {+ t.Fatalf("sub_queries = %+v, want only the one with a real sub_query", got.SubQueries)+ }+ if sq := got.SubQueries[0]; sq.SubQuery != "ok" || sq.MissingFact != "" || sq.SearchHint != "" {+ t.Errorf("absent missing_fact/search_hint must stay empty, got %+v", sq)+ }+ if got.Reasoning != "" {+ t.Errorf(`absent reasoning must stay empty, got %q`, got.Reasoning)+ }+}++// TestSCAMissingInformationIgnoresAbsentFields pins that a missing_information+// entry with no usable text is not fabricated from nil (which fmt.Sprint would+// render as "<nil>").+func TestSCAMissingInformationIgnoresAbsentFields(t *testing.T) {+ mdl := &stubJSONModel{value: map[string]any{+ "is_sufficient": false,+ "claims": []any{+ map[string]any{"claim_id": "c1", "grounded": false, "missing_information": []any{+ map[string]any{}, // both what/search_hint absent+ nil, // nil element+ map[string]any{"what": "the year"},+ }},+ },+ }}+ got := SufficientContextAgent(context.Background(), SCADeps{+ KB: &harness.Kbinfos{},+ Model: mdl,+ }, "q", []ClaimDraft{{ID: "c1", Draft: "d"}})+ mi := got.Claims["c1"].MissingInformation+ if len(mi) != 1 || mi[0].What != "the year" || mi[0].SearchHint != "" {+ t.Fatalf("missing_information = %+v, want only the real entry", mi)+ }+}++// TestRenderOverallDraftTruncatesByRune pins that the cap is applied by code+// point (Python's str[:N]), never mid-rune: a byte slice through a CJK rune+// would put invalid UTF-8 into the SCA prompt.+func TestRenderOverallDraftTruncatesByRune(t *testing.T) {+ draft := strings.Repeat("中", scaClaimsContextMax+50)+ got := renderOverallDraft([]ClaimDraft{{ID: "c1", Draft: draft}})+ if !utf8.ValidString(got) {+ t.Errorf("truncated draft is not valid UTF-8: %q", got)+ }+ if n := utf8.RuneCountInString(got); n != scaClaimsContextMax {+ t.Errorf("truncated draft has %d runes, want %d", n, scaClaimsContextMax)+ }+}++// TestRenderClaimContextBudgetIsRuneBased pins that the claim-context budget is+// counted in code points (Python's len(str)), not bytes: two CJK blocks of 20k+// runes are ~40k runes (under the 48k cap) but ~120KB bytes (far over it), so a+// byte budget would stop after the FIRST block and silently starve the SCA.+func TestRenderClaimContextBudgetIsRuneBased(t *testing.T) {+ draft := strings.Repeat("中", 20000)+ got := renderClaimContext([]ClaimDraft{+ {ID: "c1", Draft: draft},+ {ID: "c2", Draft: draft},+ }, &harness.Kbinfos{})+ if !strings.Contains(got, "Claim c1") || !strings.Contains(got, "Claim c2") {+ t.Errorf("rune-based budget must keep both CJK blocks; got %d runes", utf8.RuneCountInString(got))+ }+}++// TestRenderClaimContextTrimsOversizedBlock pins that the cap bounds the RENDERED+// context: a single claim longer than scaClaimsContextMax must be trimmed to the+// budget instead of being appended whole (which pushed the SCA prompt past the+// window the cap was sized for).+func TestRenderClaimContextTrimsOversizedBlock(t *testing.T) {+ got := renderClaimContext([]ClaimDraft{+ {ID: "c1", Draft: strings.Repeat("中", scaClaimsContextMax+5000)},+ }, &harness.Kbinfos{})+ if !utf8.ValidString(got) {+ t.Error("trimmed block is not valid UTF-8")+ }+ if n := utf8.RuneCountInString(got); n != scaClaimsContextMax {+ t.Errorf("rendered context = %d runes, want it capped at %d", n, scaClaimsContextMax)+ }+}diff --git a/internal/rag/advanced_rag/harness/tool_compiled_expansion.go b/internal/rag/advanced_rag/harness/tool_compiled_expansion.goindex ff370c54d4..cebae3a427 100644--- a/internal/rag/advanced_rag/harness/tool_compiled_expansion.go+++ b/internal/rag/advanced_rag/harness/tool_compiled_expansion.go@@ -19,6 +19,7 @@ package harnessimport ("context""encoding/json"+ "fmt""math""sort""strings"@@ -154,11 +155,19 @@ func NewCompiledExpander(store CompiledStore, cfg CompiledScopeConfig) CompiledE// the three synthesis-page expansions (wiki_page / artifact_page / essence) —// each capped at max_chunks=5 — then re-sorts the whole kbinfos["chunks"] by// similarity so compiled chunks blend by relevance with the regular ones.+//+// A doc-tenant resolution failure aborts the expansion and is returned rather+// than swallowed: Python's per-doc lookup raises out of _expand_with_compiled+// (compiled_expansion.py:224) and the caller reports it (tool_search.go,+// "compiled expansion failed") while the already-retrieved chunks stay in place.func (e *compiledExpander) Expand(ctx context.Context, kb *Kbinfos, query, keywords string, docScope []string) error {if e == nil || e.store == nil || kb == nil {return nil}- before := len(kb.Chunks)+ // Counted from what was actually pooled, not from a len(before)/len(after)+ // delta: another session's appends land in the same pool and would be+ // counted here as this expansion's contribution.+ expanded := 0seen := make(map[string]bool, len(kb.Chunks))for _, c := range kb.Chunks {if id := ChunkIDOf(c); id != "" {@@ -170,44 +179,71 @@ func (e *compiledExpander) Expand(ctx context.Context, kb *Kbinfos, query, keywomatch = strings.TrimSpace(keywords)}+ // Admit one expansion batch. UNCAPPED, like Python's expansion (it appends+ // straight to kbinfos, which is what pushes the pool past _EVIDENCE_POOL_CAP),+ // but under the pool lock and deduped against the LIVE pool — the per-call+ // `seen` snapshot it is also gated on cannot see a concurrent session's+ // appends. Returns how many chunks were actually new.+ admitExpansion := func(chunks []map[string]any) int {+ added := 0+ kb.Admit(func(p *PoolAdmitter) {+ for _, c := range chunks {+ if p.Add(c) {+ added+++ }+ }+ })+ expanded += added+ return added+ }+scopes := e.kgScopes(docScope)for _, sc := range scopes {// 1-hop entity-graph expansion, per template kind (Python L71-89):// compile_kwd empty, template_kind selects the template.for _, tk := range compiledEntityKinds {- chunks := e.expandCompiledStrategy(ctx, sc, match, seen, "", tk.kind, 5)- if len(chunks) > 0 {- kb.Chunks = append(kb.Chunks, chunks...)- _LOG.Printf("[Compiled expand] %s: +%d chunks", tk.label, len(chunks))+ chunks, err := e.expandCompiledStrategy(ctx, sc, match, seen, "", tk.kind, 5)+ if err != nil {+ return err+ }+ if n := admitExpansion(chunks); n > 0 {+ _LOG.Printf("[Compiled expand] %s: +%d chunks", tk.label, n)}}// Tree structure graph, selected by compile_kwd (Python L91-104):// template_kind empty, compile_kwd="tree".- chunks := e.expandCompiledStrategy(ctx, sc, match, seen, "tree", "", 5)- if len(chunks) > 0 {- kb.Chunks = append(kb.Chunks, chunks...)- _LOG.Printf("[Compiled expand] tree: +%d chunks", len(chunks))+ chunks, err := e.expandCompiledStrategy(ctx, sc, match, seen, "tree", "", 5)+ if err != nil {+ return err+ }+ if n := admitExpansion(chunks); n > 0 {+ _LOG.Printf("[Compiled expand] tree: +%d chunks", n)}// Synthesis pages — standalone rendered articles, searched directly// (Python L106-125).for _, ck := range compiledSynthesisKinds {- chunks := e.expandWikiPageStrategy(ctx, sc, match, seen, ck.kind, 5)- if len(chunks) > 0 {- kb.Chunks = append(kb.Chunks, chunks...)- _LOG.Printf("[Compiled expand] %s: +%d chunks", ck.label, len(chunks))+ chunks, err := e.expandWikiPageStrategy(ctx, sc, match, seen, ck.kind, 5)+ if err != nil {+ return err+ }+ if n := admitExpansion(chunks); n > 0 {+ _LOG.Printf("[Compiled expand] %s: +%d chunks", ck.label, n)}}}// Re-sort so compiled-expansion chunks blend by similarity with regular ones- // (Python L127-130).- if len(kb.Chunks) > 0 {- sort.SliceStable(kb.Chunks, func(i, j int) bool {- return similarityOf(kb.Chunks[i]) > similarityOf(kb.Chunks[j])- })- }+ // (Python L127-130). Under the pool lock: this PERMUTES the shared slice, so+ // it must not run while another session reads or appends it.+ kb.Admit(func(*PoolAdmitter) {+ if len(kb.Chunks) > 0 {+ sort.SliceStable(kb.Chunks, func(i, j int) bool {+ return similarityOf(kb.Chunks[i]) > similarityOf(kb.Chunks[j])+ })+ }+ })- _LOG.Printf("[Hybrid search] Compiled expansion added %d chunks.", len(kb.Chunks)-before)+ _LOG.Printf("[Hybrid search] Compiled expansion added %d chunks.", expanded)return nil}@@ -237,6 +273,10 @@ func (e *compiledExpander) kgScopes(docScope []string) []compiledScope {func (e *compiledExpander) searchCompiledRows(ctx context.Context, sc compiledScope, matchText string, topN int, filters map[string][]string) []map[string]any {rows, err := e.store.SearchCompiled(ctx, sc.kbID, sc.tenantID, sc.docIDs, filters, matchText, topN)if err != nil {+ // Python logs the failed search and carries on with no rows+ // (compiled_expansion.py:211): the expansion is an enrichment, so a+ // missing compiled leg must not fail the regular retrieval.+ _LOG.Printf("[Compiled expand] compiled-row search failed (kb=%s tenant=%s); treating as no rows: %v", sc.kbID, sc.tenantID, err)return nil}return rows@@ -247,11 +287,11 @@ func (e *compiledExpander) searchCompiledRows(ctx context.Context, sc compiledSc// hop 1-hop through adjacent relations (forward + backward), collect neighbour// entity names, look up their source_chunk_ids by exact name_kwd, then load the// referenced source chunks, deduped and capped at maxChunks.-func (e *compiledExpander) expandCompiledStrategy(ctx context.Context, sc compiledScope, match string, seen map[string]bool, compileKwd, templateKwd string, maxChunks int) []map[string]any {+func (e *compiledExpander) expandCompiledStrategy(ctx context.Context, sc compiledScope, match string, seen map[string]bool, compileKwd, templateKwd string, maxChunks int) ([]map[string]any, error) {// -- 1. Seed entities (embedding match, top 5) --seedRows := e.searchCompiledRows(ctx, sc, match, 5, seedFilters("entity", compileKwd, templateKwd))if len(seedRows) == 0 {- return nil+ return nil, nil}seedNames := make(map[string]bool)for _, r := range seedRows {@@ -260,7 +300,7 @@ func (e *compiledExpander) expandCompiledStrategy(ctx context.Context, sc compil}}if len(seedNames) == 0 {- return nil+ return nil, nil}// -- 2. Adjacent relations (outgoing + incoming), top 50 each --@@ -296,7 +336,7 @@ func (e *compiledExpander) expandCompiledStrategy(ctx context.Context, sc compil}}if len(neighbourNames) == 0 {- return nil+ return nil, nil}// -- 4. Neighbour entity source_chunk_ids (exact name_kwd lookup) --@@ -330,11 +370,11 @@ func (e *compiledExpander) expandCompiledStrategy(ctx context.Context, sc compil// synthesis-compiled pages of one compile_kwd directly, collect their// source_chunk_ids, then load the referenced source chunks. Pages rank high, so// each loaded chunk is assigned similarity=0.9 unless it already carries one.-func (e *compiledExpander) expandWikiPageStrategy(ctx context.Context, sc compiledScope, match string, seen map[string]bool, compileKwd string, maxChunks int) []map[string]any {+func (e *compiledExpander) expandWikiPageStrategy(ctx context.Context, sc compiledScope, match string, seen map[string]bool, compileKwd string, maxChunks int) ([]map[string]any, error) {// -- 1. Search synthesis pages (no knowledge_graph_kwd filter) --rows := e.searchSynthesisPages(ctx, sc, match, compileKwd, 5)if len(rows) == 0 {- return nil+ return nil, nil}// -- 2. Collect source_chunk_ids from matching pages --@@ -353,17 +393,20 @@ func (e *compiledExpander) expandWikiPageStrategy(ctx context.Context, sc compil}}if len(order) == 0 {- return nil+ return nil, nil}// -- 3. Load chunks, assign high similarity for priority ranking --- loaded := e.loadByDoc(ctx, sc, order, byDoc, seen, maxChunks)+ loaded, err := e.loadByDoc(ctx, sc, order, byDoc, seen, maxChunks)+ if err != nil {+ return nil, err+ }for _, c := range loaded {if _, has := c["similarity"]; !has {c["similarity"] = 0.9 // wiki pages rank high}}- return loaded+ return loaded, nil}// searchSynthesisPages mirrors Python _search_synthesis_pages: find synthesis@@ -381,7 +424,11 @@ func (e *compiledExpander) searchSynthesisPages(ctx context.Context, sc compiled// global maxChunks cap and the seen-id dedup set. Returns newly added chunks in// doc order. Mirrors Python's per-doc _load_chunks_for_doc loop and its// max_chunks break/limit accounting.-func (e *compiledExpander) loadByDoc(ctx context.Context, sc compiledScope, order []string, byDoc map[string][]string, seen map[string]bool, maxChunks int) []map[string]any {+//+// A doc whose owner cannot be resolved is dropped (the documented per-row rule),+// but a doc-tenant resolution FAILURE is returned: it is a transport/database+// error, not "no row resolved", and Python surfaces it as an exception.+func (e *compiledExpander) loadByDoc(ctx context.Context, sc compiledScope, order []string, byDoc map[string][]string, seen map[string]bool, maxChunks int) ([]map[string]any, error) {// Python _load_chunks_for_doc resolves EACH row's doc_id to its owning// (kb, tenant) — not the searched scope — and loads nothing when that// resolution fails (merged dataset rows carry a pseudo/empty doc_id). Loading@@ -395,9 +442,16 @@ func (e *compiledExpander) loadByDoc(ctx context.Context, sc compiledScope, ordeids = append(ids, docID)}}- if resolved, err := e.cfg.DocTenantResolver.ResolveDocTenants(ctx, ids); err == nil {- owners = resolved+ resolved, err := e.cfg.DocTenantResolver.ResolveDocTenants(ctx, ids)+ if err != nil {+ // Python resolves one doc at a time and lets a lookup failure raise out+ // of _load_chunks_for_doc (compiled_expansion.py:224): nothing is+ // loaded, and the failure is loud. Return it so the caller reports it+ // instead of letting a transport error masquerade as the documented+ // per-row drop below.+ return nil, fmt.Errorf("doc-tenant resolution failed for %d doc(s): %w", len(ids), err)}+ owners = resolved}var out []map[string]anyfor _, docID := range order {@@ -419,6 +473,9 @@ func (e *compiledExpander) loadByDoc(ctx context.Context, sc compiledScope, orde}rows, err := e.store.LoadChunks(ctx, kbID, tenantID, cids)if err != nil {+ // Python logs the failed load per doc (compiled_expansion.py:248) and+ // drops only that doc; the remaining docs still load.+ _LOG.Printf("[Compiled expand] failed to load chunks for doc_id=%s: %v", docID, err)continue}for _, c := range rows {@@ -430,7 +487,7 @@ func (e *compiledExpander) loadByDoc(ctx context.Context, sc compiledScope, ordeout = append(out, c)}}- return out+ return out, nil}// seedNameFromRow mirrors Python L293-300: parse content_with_weight as JSON anddiff --git a/internal/rag/advanced_rag/harness/tool_compiled_expansion_test.go b/internal/rag/advanced_rag/harness/tool_compiled_expansion_test.goindex abe2230137..46c5748eee 100644--- a/internal/rag/advanced_rag/harness/tool_compiled_expansion_test.go+++ b/internal/rag/advanced_rag/harness/tool_compiled_expansion_test.go@@ -17,7 +17,10 @@package harnessimport (+ "bytes""context"+ "errors"+ "log""strings""testing"@@ -36,6 +39,9 @@ import (type stubCompiledStore struct {rows []map[string]anychunks map[string]string+ // loadErrByChunk makes LoadChunks fail when one of the requested ids is+ // listed, standing in for a per-doc store error.+ loadErrByChunk map[string]error}func (s *stubCompiledStore) SearchCompiled(_ context.Context, _, _ string, _ []string, filters map[string][]string, matchText string, topN int) ([]map[string]any, error) {@@ -59,6 +65,11 @@ func (s *stubCompiledStore) SearchCompiled(_ context.Context, _, _ string, _ []s}func (s *stubCompiledStore) LoadChunks(_ context.Context, _, _ string, chunkIDs []string) ([]map[string]any, error) {+ for _, id := range chunkIDs {+ if err, ok := s.loadErrByChunk[id]; ok {+ return nil, err+ }+ }var out []map[string]anyfor _, id := range chunkIDs {if content, ok := s.chunks[id]; ok {@@ -429,3 +440,111 @@ func TestExpandCapsMaxChunksAndSkipsSeen(t *testing.T) {}}}++// mapTenantResolver resolves each doc id to its own owner and can fail the whole+// batch, standing in for the batched DB lookup (a per-doc lookup in Python).+type mapTenantResolver struct {+ owners map[string]DocTenant+ err error+}++func (m mapTenantResolver) ResolveDocTenants(_ context.Context, docIDs []string) (map[string]DocTenant, error) {+ if m.err != nil {+ return nil, m.err+ }+ out := make(map[string]DocTenant, len(docIDs))+ for _, d := range docIDs {+ if o, ok := m.owners[d]; ok {+ out[d] = o+ }+ }+ return out, nil+}++// TestCompiledExpanderReportsDocTenantResolutionFailure pins the two rules apart:+// a row whose owner cannot be resolved is dropped silently (see the test above),+// but a FAILED batched lookup is a transport error — Python lets it raise out of+// _load_chunks_for_doc, so nothing is loaded and the failure is reported instead+// of masquerading as "no row resolved".+func TestCompiledExpanderReportsDocTenantResolutionFailure(t *testing.T) {+ store := &ownerRecordingStore{stubCompiledStore: &stubCompiledStore{+ rows: []map[string]any{+ {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",+ "name_kwd": "Culdcept", "content_with_weight": `{"name":"Culdcept"}`, "doc_id": "seedDoc"},+ {"knowledge_graph_kwd": "relation", "compilation_template_kind_kwd": "knowledge_graph",+ "from_entity_kwd": "Culdcept", "to_entity_kwd": "OmiyaSoft", "doc_id": "seedDoc"},+ {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",+ "name_kwd": "OmiyaSoft", "content_with_weight": `{"name":"OmiyaSoft"}`,+ "source_chunk_ids": []string{"s1"}, "doc_id": "docA"},+ },+ chunks: map[string]string{"s1": "a"},+ }}+ exp := NewCompiledExpander(store, CompiledScopeConfig{+ DatasetIDs: []string{"kb1"},+ TenantID: "t1",+ KBs: []*entity.Knowledgebase{{ID: "kb1", TenantID: "t1"}},+ DocTenantResolver: mapTenantResolver{err: errors.New("db unavailable")},+ }).(*compiledExpander)++ kb := &Kbinfos{}+ err := exp.Expand(context.Background(), kb, "culdcept", "", nil)+ if err == nil {+ t.Fatal("Expand must report the resolver failure instead of silently loading nothing")+ }+ if !strings.Contains(err.Error(), "doc-tenant resolution failed for 1 doc(s)") {+ t.Errorf("error = %q, want it to name the failed batch and its size", err)+ }+ if len(store.loads) != 0 {+ t.Errorf("LoadChunks calls = %+v, want none after a resolver failure", store.loads)+ }+ if len(kb.Chunks) != 0 {+ t.Errorf("chunks = %v, want none", compiledIDs(kb.Chunks))+ }+}++// TestCompiledExpanderLogsAndKeepsOtherDocsWhenOneLoadFails mirrors Python's+// per-doc load: the failing doc is logged (compiled_expansion.py:248) and+// dropped, while the remaining docs still contribute.+func TestCompiledExpanderLogsAndKeepsOtherDocsWhenOneLoadFails(t *testing.T) {+ var buf bytes.Buffer+ prevLog := _LOG+ _LOG = log.New(&buf, "", 0)+ t.Cleanup(func() { _LOG = prevLog })++ store := &ownerRecordingStore{stubCompiledStore: &stubCompiledStore{+ rows: []map[string]any{+ {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",+ "name_kwd": "Culdcept", "content_with_weight": `{"name":"Culdcept"}`, "doc_id": "seedDoc"},+ {"knowledge_graph_kwd": "relation", "compilation_template_kind_kwd": "knowledge_graph",+ "from_entity_kwd": "Culdcept", "to_entity_kwd": "OmiyaSoft", "doc_id": "seedDoc"},+ {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",+ "name_kwd": "OmiyaSoft", "content_with_weight": `{"name":"OmiyaSoft"}`,+ "source_chunk_ids": []string{"s1"}, "doc_id": "docA"},+ {"knowledge_graph_kwd": "entity", "compilation_template_kind_kwd": "knowledge_graph",+ "name_kwd": "OmiyaSoft", "content_with_weight": `{"name":"OmiyaSoft"}`,+ "source_chunk_ids": []string{"s2"}, "doc_id": "docB"},+ },+ chunks: map[string]string{"s1": "a", "s2": "b"},+ loadErrByChunk: map[string]error{"s1": errors.New("store unavailable")},+ }}+ exp := NewCompiledExpander(store, CompiledScopeConfig{+ DatasetIDs: []string{"kb1"},+ TenantID: "t1",+ KBs: []*entity.Knowledgebase{{ID: "kb1", TenantID: "t1"}},+ DocTenantResolver: mapTenantResolver{owners: map[string]DocTenant{+ "docA": {KBID: "kbA", TenantID: "tA"},+ "docB": {KBID: "kbB", TenantID: "tB"},+ }},+ }).(*compiledExpander)++ kb := &Kbinfos{}+ if err := exp.Expand(context.Background(), kb, "culdcept", "", nil); err != nil {+ t.Fatalf("Expand: %v", err)+ }+ if got := compiledIDs(kb.Chunks); len(got) != 1 || got[0] != "s2" {+ t.Errorf("expanded = %v, want only s2 (docA's load failed)", got)+ }+ if !strings.Contains(buf.String(), "failed to load chunks for doc_id=docA") {+ t.Errorf("log = %q, want the failing doc named", buf.String())+ }+}diff --git a/internal/rag/advanced_rag/harness/tool_executor.go b/internal/rag/advanced_rag/harness/tool_executor.goindex 502361baf9..36bde2d7b8 100644--- a/internal/rag/advanced_rag/harness/tool_executor.go+++ b/internal/rag/advanced_rag/harness/tool_executor.go@@ -333,7 +333,7 @@ func (e *searchExecutor) navigateTree(ctx context.Context, args map[string]any)// It stays in the harness package because it depends on the agentic Retriever /// RetrieveRequest; the routing algorithm itself lives in internal/service/nav.func chunkAggRetrieveFrom(r Retriever) nav.ChunkRetriever {- return func(ctx context.Context, tenantID, kbID, query string, docScope []string, topN int, _ float64) []map[string]any {+ return func(ctx context.Context, tenantID, kbID, query string, docScope []string, topN int, _ float64) ([]map[string]any, error) {chunks, err := r.Retrieve(ctx, RetrieveRequest{Query: query,DatasetIDs: []string{kbID},@@ -341,10 +341,13 @@ func chunkAggRetrieveFrom(r Retriever) nav.ChunkRetriever {DocScope: docScope,TopN: topN,})- if err != nil || len(chunks) == 0 {- return nil+ if err != nil {+ // Surface the failure instead of folding it into an empty result:+ // the router reports it as an error so the orchestrator takes its+ // fallback rather than telling the model the tree routed nothing.+ return nil, err}- return chunks+ return chunks, nil}}@@ -502,6 +505,17 @@ func argString(args map[string]any, key string) string {return s}+// evidencePoolCap is the hard cap on the shared evidence pool+// (kbinfos["chunks"]). Mirrors Python _EVIDENCE_POOL_CAP (=60, the same ceiling+// as _MAX_SNIPPET_POOL / _SCA_VIEW_CAP): once the pool is saturated, further+// admits cannot reach the SCA view or improve the answer, so the action session+// stops admitting chunks (observed pools otherwise grew to ~106).+const evidencePoolCap = 60++// The cap check and its "pool FULL" line now live on PoolAdmitter.Full, where+// the pool lock is held (see kbinfos.go): the check must not read len(Chunks)+// while another session appends.+// search runs one retrieval call for a tool invocation.//// Python's retrieve/search_chunks both funnel into tools/search.py, differing@@ -549,13 +563,10 @@ func (e *searchExecutor) search(ctx context.Context, name string, args map[strinnewChunks := 0// Per-call admittance state, mirroring Python _run_search/_admit_evidence// (action_session.py:_run_search): `seen` dedups chunks ACROSS the queries of- // this one call; kbSeen holds the shared pool's existing identities- // (Python `kb_seen`).+ // this one call. The pool-side dedup is Kbinfos.Admit's job, against the LIVE+ // pool — a per-call snapshot of it (Python's `kb_seen`) is exact only while+ // nothing can interleave, and another session appending makes it stale.seen := map[string]bool{}- kbSeen := make(map[string]bool, len(e.deps.KB.Chunks))- for _, c := range e.deps.KB.Chunks {- kbSeen[chunkKey(c)] = true- }for _, q := range queries {// Per-tool top_n (mirrors Python action_session: retrieve=10,// search_chunks=20). They were previously collapsed onto e.req.TopN,@@ -571,10 +582,11 @@ func (e *searchExecutor) search(ctx context.Context, name string, args map[strin// global switch can no longer disable the semantic leg.//// Python's search_chunks tool ALWAYS enables compiled-structure expansion- // (action_session.py:execute_tool passes use_compiled=True); retrieve/grep_search- // never do. So Go mirrors that: compiled is on for search_chunks and off- // for every other retrieve-family tool, independent of e.req.UseCompiled- // (which gates the L1 direct retrieve, not the action_session tool loop).+ // in ALL modes, not just high (action_session.py:execute_tool passes+ // use_compiled=True); retrieve/grep_search never do. So Go mirrors that:+ // compiled is on for search_chunks and off for every other retrieve-family+ // tool, independent of e.req.UseCompiled (which gates the L1 direct+ // retrieve, not the action_session tool loop).var searchFn func(context.Context, SearchDeps, SearchParams) ([]map[string]any, []map[string]any)useCompiled := name == "search_chunks"if useCompiled {@@ -627,24 +639,33 @@ func (e *searchExecutor) search(ctx context.Context, name string, args map[strin// id, the chunk ID as the evidence reference (Python's `ids` holds ids,// not pool positions), and only chunks NEW to the shared pool appended// to it — so REDUNDANT means "nothing new", not "nothing returned".- for _, c := range chunks {- cid := ChunkIDOf(c)- if seen[cid] {- continue- }- seen[cid] = true- evidenceIDs = append(evidenceIDs, cid)- payload = append(payload, passageFromChunk(c))- // Pool identity uses chunkKey (Go's stable key): Python's _chunk_key- // falls back to id(ck) — the dict's address — so an equivalent- // re-retrieved chunk never matches and is appended again.- key := chunkKey(c)- if !kbSeen[key] {- kbSeen[key] = true- e.deps.KB.Chunks = append(e.deps.KB.Chunks, c)- newChunks+++ //+ // ONE query's batch is one critical section: Python's per-query loop has+ // no await (the awaits sit in the outer query loop, :691-700), so asyncio+ // cannot interleave two sessions' batches. Locking per chunk would let+ // them interleave into pool orders Python can never produce.+ e.deps.KB.Admit(func(p *PoolAdmitter) {+ for _, c := range chunks {+ // Python _admit_evidence early-stops at the top once the shared pool+ // reaches the cap, BEFORE the per-call dedup.+ if p.Full() {+ continue+ }+ cid := ChunkIDOf(c)+ if seen[cid] {+ continue+ }+ seen[cid] = true+ evidenceIDs = append(evidenceIDs, cid)+ payload = append(payload, passageFromChunk(c))+ // Pool identity uses chunkKey (Go's stable key): Python's _chunk_key+ // falls back to id(ck) — the dict's address — so an equivalent+ // re-retrieved chunk never matches and is appended again.+ if p.Add(c) {+ newChunks+++ }}- }+ })// Pool this search's doc_aggs (skipped when it returned no chunks, like// _merge_kbinfos).e.deps.KB.MergeDocAggs(aggs)@@ -920,14 +941,18 @@ type RuntimeRetriever struct{}func (r *RuntimeRetriever) Retrieve(ctx context.Context, req RetrieveRequest) ([]map[string]any, error) {svc := runtime.GetRetrievalService()chunks, err := svc.Search(ctx, nil, runtime.RetrievalRequest{- Query: req.Query,- DatasetIDs: req.DatasetIDs,- DocScope: req.DocScope,- TopN: req.TopN,- TopK: req.TopK,- RerankCandidatesCount: req.RerankCandidatesCount,- SimilarityThreshold: &req.SimilarityThreshold,- KeywordsSimilarityWeight: &req.KeywordsSimilarityWeight,+ Query: req.Query,+ DatasetIDs: req.DatasetIDs,+ DocScope: req.DocScope,+ TopN: req.TopN,+ TopK: req.TopK,+ RerankCandidatesCount: req.RerankCandidatesCount,+ // Passed through as pointers: nil (the caller did not supply one) must+ // stay nil so the retrieval service keeps its own default, while a zero+ // is a real override. Taking the address of a zero value here forced+ // "threshold 0 / full vector weight" onto every caller that omitted them.+ SimilarityThreshold: req.SimilarityThreshold,+ KeywordsSimilarityWeight: req.KeywordsSimilarityWeight,TenantID: req.TenantID,RankFeature: req.RankFeature,// ExcludeCompiled maps Python hybrid_search'sdiff --git a/internal/rag/advanced_rag/harness/tool_executor_test.go b/internal/rag/advanced_rag/harness/tool_executor_test.goindex 2af207ce09..50e93107a1 100644--- a/internal/rag/advanced_rag/harness/tool_executor_test.go+++ b/internal/rag/advanced_rag/harness/tool_executor_test.go@@ -17,6 +17,7 @@ package harnessimport ("bytes""context"+ "errors""fmt""log""strings"@@ -28,12 +29,17 @@ import ()// stubRetrievalService returns a fixed set of child chunks so the harness-// retireval path can be exercised without a live search backend.+// retireval path can be exercised without a live search backend. lastReq, when+// set, receives the request the harness handed down.type stubRetrievalService struct {- chunks []runtime.RetrievalChunk+ chunks []runtime.RetrievalChunk+ lastReq *runtime.RetrievalRequest}-func (s stubRetrievalService) Search(_ context.Context, _ *gorm.DB, _ runtime.RetrievalRequest) ([]runtime.RetrievalChunk, error) {+func (s stubRetrievalService) Search(_ context.Context, _ *gorm.DB, req runtime.RetrievalRequest) ([]runtime.RetrievalChunk, error) {+ if s.lastReq != nil {+ *s.lastReq = req+ }return s.chunks, nil}@@ -52,6 +58,78 @@ func (s stubDocEngine) GetChunk(_ context.Context, _, chunkID string, _ []stringreturn nil, fmt.Errorf("parent %s not found", chunkID)}+// TestRuntimeRetrieverPreservesUnsetControls pins the presence semantics of the+// retrieval controls: an omitted threshold/weight must reach the retrieval+// service as nil so it keeps its own default, while an explicit zero is a real+// override. Taking the address of a zero value used to force "no threshold+// floor + the vector leg at full weight" onto every caller that omitted them.+func TestRuntimeRetrieverPreservesUnsetControls(t *testing.T) {+ prev := runtime.GetRetrievalService()+ var got runtime.RetrievalRequest+ runtime.SetRetrievalService(stubRetrievalService{lastReq: &got})+ t.Cleanup(func() { runtime.SetRetrievalService(prev) })++ r := &RuntimeRetriever{}+ if _, err := r.Retrieve(context.Background(), RetrieveRequest{Query: "q", DatasetIDs: []string{"kb-1"}}); err != nil {+ t.Fatalf("Retrieve: %v", err)+ }+ if got.SimilarityThreshold != nil || got.KeywordsSimilarityWeight != nil {+ t.Errorf("omitted controls = %v / %v, want nil so the service keeps its defaults",+ got.SimilarityThreshold, got.KeywordsSimilarityWeight)+ }++ threshold, weight := 0.35, 0.3+ if _, err := r.Retrieve(context.Background(), RetrieveRequest{+ Query: "q",+ SimilarityThreshold: &threshold,+ KeywordsSimilarityWeight: &weight,+ }); err != nil {+ t.Fatalf("Retrieve: %v", err)+ }+ if got.SimilarityThreshold == nil || *got.SimilarityThreshold != threshold {+ t.Errorf("SimilarityThreshold = %v, want %v", got.SimilarityThreshold, threshold)+ }+ if got.KeywordsSimilarityWeight == nil || *got.KeywordsSimilarityWeight != weight {+ t.Errorf("KeywordsSimilarityWeight = %v, want %v", got.KeywordsSimilarityWeight, weight)+ }+}++// TestChunkAggRetrieveLeavesControlsUnset pins the caller side: the chunk-agg+// retriever has no threshold/weight of its own, so it must omit both rather+// than pass zero overrides.+func TestChunkAggRetrieveLeavesControlsUnset(t *testing.T) {+ r := &stubRetriever{chunks: []map[string]any{{"chunk_id": "c1"}}}++ chunks, err := chunkAggRetrieveFrom(r)(context.Background(), "t1", "kb-1", "q", nil, 5, 0.9)+ if err != nil {+ t.Fatalf("chunkAggRetrieveFrom: %v", err)+ }+ if len(chunks) != 1 {+ t.Fatalf("chunks = %d, want the retrieved chunk", len(chunks))+ }+ req := r.lastReq(t)+ if req.SimilarityThreshold != nil || req.KeywordsSimilarityWeight != nil {+ t.Errorf("controls = %v / %v, want nil (zero is a valid value, not an unset marker)",+ req.SimilarityThreshold, req.KeywordsSimilarityWeight)+ }+}++// TestChunkAggRetrievePropagatesError pins the adapter half of the router+// contract: a retrieval failure must reach the router as an error, not as an+// empty chunk list (which the router would report as a successful empty route).+func TestChunkAggRetrievePropagatesError(t *testing.T) {+ wantErr := errors.New("retrieval down")+ r := &stubRetriever{err: wantErr}++ chunks, err := chunkAggRetrieveFrom(r)(context.Background(), "t1", "kb-1", "q", nil, 5, 0.9)+ if !errors.Is(err, wantErr) {+ t.Fatalf("err = %v, want %v", err, wantErr)+ }+ if chunks != nil {+ t.Errorf("chunks = %v, want nil alongside the error", chunks)+ }+}+// TestRuntimeRetrieverPromotesChildrenToParent verifies that// RuntimeRetriever.Retrieve threads child chunks through retrieval_by_children:// two child fragments sharing a mom_id collapse into the single parent chunk,@@ -343,6 +421,39 @@ func TestListChunksCapsDeepReadAndOutput(t *testing.T) {}}+// TestEvidencePoolCapStopsAdmitting pins the PR's _EVIDENCE_POOL_CAP early-stop:+// once the shared pool reaches the cap, a search admits no further chunk, so its+// outcome collapses to MISS (nothing new was admitted) — a saturated session+// stops bloating the pool beyond what the SCA view can read.+func TestEvidencePoolCapStopsAdmitting(t *testing.T) {+ pre := make([]map[string]any, 0, evidencePoolCap)+ for i := 0; i < evidencePoolCap; i++ {+ pre = append(pre, map[string]any{"chunk_id": fmt.Sprintf("pre-%d", i), "content": "old"})+ }+ deps, kb := newTestSearchDeps(&stubRetriever{chunks: []map[string]any{{"content": "hit", "chunk_id": "c1"}}})+ kb.Chunks = pre+ ex := NewSearchExecutor(deps, RunRequest{DatasetIDs: []string{"kb1"}})+ oc, err := ex.Execute(context.Background(), "retrieve", map[string]any{"query": "q"})+ if err != nil {+ t.Fatalf("retrieve: %v", err)+ }+ if oc.Status != StatusMiss {+ t.Errorf("status = %s, want %s (a full pool admits nothing)", oc.Status, StatusMiss)+ }+ if len(kb.Chunks) != evidencePoolCap {+ t.Errorf("kb.Chunks = %d, want the pool to stay at the cap %d", len(kb.Chunks), evidencePoolCap)+ }+ // Dropping one chunk frees a slot and admission resumes.+ kb.Chunks = kb.Chunks[:evidencePoolCap-1]+ oc, err = ex.Execute(context.Background(), "retrieve", map[string]any{"query": "q"})+ if err != nil {+ t.Fatalf("retrieve after freeing a slot: %v", err)+ }+ if oc.Status != StatusOK {+ t.Errorf("status after freeing a slot = %s, want %s", oc.Status, StatusOK)+ }+}+// TestWebSearchAdmitsToPool pins the web_search parity fix: web results merge// into the SAME shared evidence pool as corpus hits (Python _exec_web_search →// _admit_evidence), so downstream formalize/compose can cite them, and thediff --git a/internal/rag/advanced_rag/harness/tool_exploration.go b/internal/rag/advanced_rag/harness/tool_exploration.goindex 00502a58bf..0359def223 100644--- a/internal/rag/advanced_rag/harness/tool_exploration.go+++ b/internal/rag/advanced_rag/harness/tool_exploration.go@@ -357,6 +357,11 @@ func resolveKGScope(deps SearchDeps, docScope, datasetIDs []string) []kgScope {key := DocTenant{KBID: owner.KBID, TenantID: owner.TenantID}byOwner[key] = append(byOwner[key], docID)}+ } else {+ // Python's per-doc lookup raises here (exploration.py:68); Go degrades+ // to the bound datasets below, so the lost owner grouping must not be+ // silent.+ _LOG.Printf("[Graph explore] doc-tenant resolution failed for %d doc(s); falling back to the bound datasets: %v", len(docScope), err)}}@@ -463,7 +468,7 @@ func ExploreGraph(ctx context.Context, deps SearchDeps, tenantID string, dataset// (1) Seeds: dense KNN (similarity>=_KG_SEED_SIM) over the scoped entity// rows, re-ranked by mention_count_int desc; falls back to keyword match// when the embedding model is unavailable.- seedRows := kgSeedSearch(ctx, de, sc.TenantID, sc.KBID, sc.Docs, text, scopeKwd, seedVec)+ seedRows := kgSeedSearch(ctx, de, sc.TenantID, sc.KBID, sc.Docs, text, scopeKwd, seedVec, deps.IndexName)var seeds []kgEntityfor _, r := range seedRows {if e, ok := kgParseEntity(r); ok {@@ -479,9 +484,9 @@ func ExploreGraph(ctx context.Context, deps SearchDeps, tenantID string, dataset}terms := endpointTerms(frontier)relRows := kgSearch(ctx, de, sc.TenantID, sc.KBID, sc.Docs, "relation", "", kgRelLimit, scopeKwd,- map[string]interface{}{"from_entity_kwd": terms}, "", 0)+ map[string]interface{}{"from_entity_kwd": terms}, "", 0, deps.IndexName)relRows = append(relRows, kgSearch(ctx, de, sc.TenantID, sc.KBID, sc.Docs, "relation", "", kgRelLimit, scopeKwd,- map[string]interface{}{"to_entity_kwd": terms}, "", 0)...)+ map[string]interface{}{"to_entity_kwd": terms}, "", 0, deps.IndexName)...)// Dedup keys: Python keeps a relation unless its source row id is// already seen (rel_rows is a dict keyed by row id), so the same// endpoint pair from *different* rows survives. Mirror that: prefer@@ -535,7 +540,7 @@ func ExploreGraph(ctx context.Context, deps SearchDeps, tenantID string, datasetlimit = len(neighFiltered)}neighRows := kgSearch(ctx, de, sc.TenantID, sc.KBID, sc.Docs, "entity", "", limit, scopeKwd,- map[string]interface{}{"name_kwd": endpointTerms(neighFiltered)}, "", 0)+ map[string]interface{}{"name_kwd": endpointTerms(neighFiltered)}, "", 0, deps.IndexName)var neighbours []kgEntityfor _, r := range neighRows {if e, ok := kgParseEntity(r); ok {@@ -674,8 +679,9 @@ func defaultSeedEncoder(ctx context.Context, tenantID, text string) []float64 {// kgSeedSearch searches the compiled KG entity rows for seeds (mirrors Python// _kg_search dense branch): dense KNN over name_kwd with similarity>=0.8,// re-ranked by mention_count_int desc, top kgSeeds. Falls back to keyword match-// when seedVec is nil (embedding model unavailable).-func kgSeedSearch(ctx context.Context, de engine.DocEngine, tenantID, kbID string, docIDs []string, text, scopeKwd string, seedVec []float64) []map[string]interface{} {+// when seedVec is nil (embedding model unavailable). indexName is the caller's+// configured index override (deps.IndexName); empty keeps ragflow_<tenantID>.+func kgSeedSearch(ctx context.Context, de engine.DocEngine, tenantID, kbID string, docIDs []string, text, scopeKwd string, seedVec []float64, indexName string) []map[string]interface{} {if seedVec != nil {dense := &types.MatchDenseExpr{VectorColumnName: fmt.Sprintf("q_%d_vec", len(seedVec)),@@ -685,11 +691,11 @@ func kgSeedSearch(ctx context.Context, de engine.DocEngine, tenantID, kbID strinTopN: kgSeedPool,ExtraOptions: map[string]interface{}{"similarity": kgSeedSim},}- rows := kgSearchRaw(ctx, de, tenantID, kbID, docIDs, "entity", scopeKwd, nil, []interface{}{dense}, "mention_count_int", kgSeedPool)+ rows := kgSearchRaw(ctx, de, tenantID, kbID, docIDs, "entity", scopeKwd, nil, []interface{}{dense}, "mention_count_int", kgSeedPool, indexName)return topMentionCount(rows, kgSeeds)}// Text fallback (mirrors Python _kg_search `embed_mdl is None` path).- return kgSearch(ctx, de, tenantID, kbID, docIDs, "entity", text, kgSeeds, scopeKwd, nil, "mention_count_int", kgSeedPool)+ return kgSearch(ctx, de, tenantID, kbID, docIDs, "entity", text, kgSeeds, scopeKwd, nil, "mention_count_int", kgSeedPool, indexName)}// topMentionCount re-ranks rows by mention_count_int desc and returns topN.@@ -719,8 +725,15 @@ func mentionCount(row map[string]interface{}) int {// kgSearchRaw is the low-level KG row search with explicit match exprs and any// extra filter keys (e.g. from_entity_kwd/to_entity_kwd/name_kwd).-func kgSearchRaw(ctx context.Context, de engine.DocEngine, tenantID, kbID string, docIDs []string, kind, scopeKwd string, extra map[string]interface{}, matchExprs []interface{}, orderDesc string, limit int) []map[string]interface{} {- idx := fmt.Sprintf("ragflow_%s", tenantID)+//+// indexName is the caller's configured index override (deps.IndexName): the KG+// rows are written to the same index the chunk loads read (ExploreGraph loads+// its evidence with indexNameFor(deps.TenantID, deps.IndexName)), so a tenant+// that overrides the name must have it honoured here too — otherwise the walk+// queries one index and loads passages from another. Empty falls back to+// ragflow_<tenantID>, which is what Python's search.index_name always returns.+func kgSearchRaw(ctx context.Context, de engine.DocEngine, tenantID, kbID string, docIDs []string, kind, scopeKwd string, extra map[string]interface{}, matchExprs []interface{}, orderDesc string, limit int, indexName string) []map[string]interface{} {+ idx := indexNameFor(tenantID, indexName)condition := map[string]interface{}{"knowledge_graph_kwd": kind}if scopeKwd != "" {condition["scope_kwd"] = scopeKwd@@ -754,7 +767,7 @@ func kgSearchRaw(ctx context.Context, de engine.DocEngine, tenantID, kbID string// kgSearch searches the compiled KG rows of one KB (mirrors Python _kg_search),// using keyword match. It only builds the MatchTextExpr (with the pool-based// TopN) and delegates the request construction to kgSearchRaw.-func kgSearch(ctx context.Context, de engine.DocEngine, tenantID, kbID string, docIDs []string, kind, text string, topN int, scopeKwd string, extra map[string]interface{}, orderDesc string, pool int) []map[string]interface{} {+func kgSearch(ctx context.Context, de engine.DocEngine, tenantID, kbID string, docIDs []string, kind, text string, topN int, scopeKwd string, extra map[string]interface{}, orderDesc string, pool int, indexName string) []map[string]interface{} {var matchExprs []interface{}if text != "" {knnTopN := topN@@ -767,7 +780,7 @@ func kgSearch(ctx context.Context, de engine.DocEngine, tenantID, kbID string, dTopN: knnTopN,}}}- return kgSearchRaw(ctx, de, tenantID, kbID, docIDs, kind, scopeKwd, extra, matchExprs, orderDesc, topN)+ return kgSearchRaw(ctx, de, tenantID, kbID, docIDs, kind, scopeKwd, extra, matchExprs, orderDesc, topN, indexName)}func kgParseEntity(row map[string]interface{}) (kgEntity, bool) {diff --git a/internal/rag/advanced_rag/harness/tool_exploration_test.go b/internal/rag/advanced_rag/harness/tool_exploration_test.gonew file mode 100644index 0000000000..fbada281c3--- /dev/null+++ b/internal/rag/advanced_rag/harness/tool_exploration_test.go@@ -0,0 +1,84 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package harness++import (+ "context"+ "testing"++ "ragflow/internal/engine"+ "ragflow/internal/engine/types"+)++// kgIndexCapturingEngine records the index names of every search it serves.+type kgIndexCapturingEngine struct {+ engine.DocEngine+ indexNames []string+}++func (e *kgIndexCapturingEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ e.indexNames = append(e.indexNames, req.IndexNames...)+ return &types.SearchResult{}, nil+}++// TestKGSearchUsesConfiguredIndexName pins that the knowledge-graph walk searches+// the SAME index the rest of the request reads: ExploreGraph loads its evidence+// with indexNameFor(deps.TenantID, deps.IndexName), so a tenant that overrides+// the index name must have it honoured by the KG rows too. Hardcoding+// ragflow_<tenantID> made the walk query one index and return passages loaded+// from another. Empty keeps the default, matching Python's search.index_name.+func TestKGSearchUsesConfiguredIndexName(t *testing.T) {+ ctx := context.Background()+ for _, tc := range []struct {+ name string+ indexName string+ want string+ }{+ {"configured override", "custom_idx", "custom_idx"},+ {"default", "", "ragflow_tenantA"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ for _, call := range []struct {+ name string+ fn func(de engine.DocEngine)+ }{+ {"kgSearchRaw", func(de engine.DocEngine) {+ kgSearchRaw(ctx, de, "tenantA", "kb1", nil, "entity", "dataset", nil, nil, "", 5, tc.indexName)+ }},+ {"kgSearch", func(de engine.DocEngine) {+ kgSearch(ctx, de, "tenantA", "kb1", nil, "entity", "q", 5, "dataset", nil, "", 0, tc.indexName)+ }},+ {"kgSeedSearch", func(de engine.DocEngine) {+ kgSeedSearch(ctx, de, "tenantA", "kb1", nil, "q", "dataset", nil, tc.indexName)+ }},+ } {+ t.Run(call.name, func(t *testing.T) {+ de := &kgIndexCapturingEngine{}+ call.fn(de)+ if len(de.indexNames) == 0 {+ t.Fatal("no search was issued")+ }+ for _, got := range de.indexNames {+ if got != tc.want {+ t.Errorf("index = %q, want %q", got, tc.want)+ }+ }+ })+ }+ })+ }+}diff --git a/internal/rag/advanced_rag/harness/tool_navigation.go b/internal/rag/advanced_rag/harness/tool_navigation.goindex aae882f620..3303aac283 100644--- a/internal/rag/advanced_rag/harness/tool_navigation.go+++ b/internal/rag/advanced_rag/harness/tool_navigation.go@@ -134,16 +134,35 @@ func loadStructureGraph(ctx context.Context, indexName, docID string, kinds map[kindList = append(kindList, k)}rawEntities, rawRels := ParseCompiledStructure(rows, kindList)+ return structureGraphFromRaw(rawEntities, rawRels)+}++// structureGraphFromRaw maps what ParseCompiledStructure returns onto the typed+// entities/relations the drill-down consumes.+//+// A field the compiled payload omits must read as ABSENT, not as its Go+// rendering: fmt.Sprint(nil) is the non-empty string "<nil>", which survives+// navTypeOr's empty check and reaches the model as "- Name (<nil>): <nil>" with+// a phantom edge to "<nil>". Python renders "- Name (other)" and no description,+// because `(e.get("type") or "other")` / `(e.get("description") or "")` treat a+// missing key as falsy (navigation.py:1714-1715), and it DROPS a relation with a+// missing endpoint instead of keeping it (navigation.py:1783-1786). Leaving the+// field empty is also what lets the renderers apply Python's defaults —+// "other" for a type (navigation.py:1714), "related_to" for a relation type+// (navigation.py:1788).+func structureGraphFromRaw(rawEntities, rawRels []map[string]any) ([]structureEntity, []structureRel) {var out []structureEntityfor _, e := range rawEntities {name, _ := e["name"].(string)if strings.TrimSpace(name) == "" {continue}+ typ, _ := e["type"].(string)+ desc, _ := e["description"].(string)se := structureEntity{Name: name,- Type: fmt.Sprint(e["type"]),- Description: fmt.Sprint(e["description"]),+ Type: typ,+ Description: desc,}if v, ok := e["_vec"].([]float64); ok {se.Vec = v@@ -159,16 +178,19 @@ func loadStructureGraph(ctx context.Context, indexName, docID string, kinds map[}var rels []structureRelfor _, r := range rawRels {- p := strings.TrimSpace(fmt.Sprint(r["from"]))- c := strings.TrimSpace(fmt.Sprint(r["to"]))+ pRaw, _ := r["from"].(string)+ cRaw, _ := r["to"].(string)+ p := strings.TrimSpace(pRaw)+ c := strings.TrimSpace(cRaw)// Python _build_toc_tree skips empty or self-loop relations.if p == "" || c == "" || p == c {continue}+ relType, _ := r["type"].(string)rels = append(rels, structureRel{from: p,to: c,- relType: fmt.Sprint(r["type"]),+ relType: relType,})}return out, rels@@ -333,9 +355,23 @@ type NavResult struct {// A query-level miss is not a structure absence.func (n NavResult) HasStructure() bool { return n.EmptyReason != ReasonNoStructure }-// navEmpty returns an empty NavResult with the given reason and matching status.+// navEmpty returns an empty NavResult carrying Python's <tree_navigation> text:+// count="0" plus an error="<label>" attribute, omitted when label is empty —+// exactly the shapes navigation.py builds (:877 "no retriever", :880 "query is+// required", :899 no attribute at all for a query-level miss). That text is not+// what the model reads for an empty result — both sides substitute their own+// note for every empty_reason (action_session.py:869-876, tool_executor.go:402)+// — but the NavResult must still carry it. The status is derived from the reason+// by ReasonStatus.func navEmpty(reason, label string) NavResult {- return NavResult{EmptyReason: reason}+ attr := ""+ if label != "" {+ attr = ` error="` + XMLEscape(label) + `"`+ }+ return NavResult{+ EmptyReason: reason,+ Text: "<tree_navigation count=\"0\"" + attr + ">\n</tree_navigation>",+ }}// ---------------------------------------------------------------------------@@ -391,7 +427,9 @@ func NavigateTree(ctx context.Context, router NavTreeRouter, in NavTreeInput) Nareturn navEmpty(ReasonBadArgs, "query is required")}if router == nil {- return navEmpty(ReasonInfra, "no nav tree router")+ // Python's counterpart is a missing retriever, and the error label is its+ // string verbatim (navigation.py:877): the router IS the retrieval backend.+ return navEmpty(ReasonInfra, "no retriever")}// Descend on the topic + keywords: routing descends on similarity and does// not require keyword hits, so keywords only enrich the query.@@ -429,6 +467,18 @@ func NavigateTree(ctx context.Context, router NavTreeRouter, in NavTreeInput) Na}}if !anyTree {+ if anyError != nil {+ // A backend failure is an INFRASTRUCTURE error, not a dataset verdict:+ // no read succeeded, so "this dataset has no tree" would be a verdict+ // drawn from zero evidence — and Route() documents that contract+ // itself ("(nil, nil) when the dataset has no compiled tree at all",+ // so a non-nil error is a failure, not an absence). Python never+ // reaches no_structure from a failed descent either: its nav-tree+ // route yields only infra / bad_args / no_doc (navigation.py:877-899),+ // while no_structure belongs to the STRUCTURE path, where a successful+ // read found zero entities (:1046).+ return navEmpty(ReasonInfra, "nav tree descent failed")+ }// Every dataset lacks a compiled tree — a DATASET-level fact, so the// caller may disable the tool for the session.return navEmpty(ReasonNoStructure, "no compiled navigation tree")@@ -436,9 +486,10 @@ func NavigateTree(ctx context.Context, router NavTreeRouter, in NavTreeInput) Naif len(ordered) == 0 {// Structure exists but THIS query reached nothing: a query-level miss.// The dataset may still have a tree a better-formed query would hit.- return navEmpty(ReasonNoDoc, "no document routed")+ // No error attribute: Python's query-level miss carries none+ // (navigation.py:899), unlike infra/bad_args.+ return navEmpty(ReasonNoDoc, "")}- _ = anyError// Cap after dedup, preserving the score order.if len(ordered) > navTreeMaxDocs {@@ -982,6 +1033,8 @@ func drillKeptNodes(qvec []float64, terms []string, nodes []structureNode, relsfrontier = nextdepth++}+ // Order-independent (see drillWithAncestors): the ancestor closure is the same+ // SET whatever order the names are visited in.for name := range keptNames {cur := parents[name]guard := 0@@ -992,7 +1045,7 @@ func drillKeptNodes(qvec []float64, terms []string, nodes []structureNode, rels}}var kept []structureNode- for n := range keptNames {+ for _, n := range sortedKeptNames(keptNames) {if e, ok := byName[n]; ok {kept = append(kept, e)}@@ -1126,9 +1179,34 @@ type chunkHit struct {score float64}+// sortedKeptNames returns a kept-name set as a sorted slice.+//+// The kept set is a map because it is used for membership tests, but three+// order-sensitive outputs are materialised from it: the rendered outline+// (kept[:structMaxNodes]), chunkPaths (first writer wins for a chunk covered by+// several nodes) and collectChunkIDs(kept, 32) — the chunk list whose snippets+// reach the model. Python iterates the kept_names SET (navigation.py:1660, 1673 and+// _drill_kept_nodes:1594), so its order is arbitrary and, because str hashing is+// randomised per process, not even reproducible across runs: there is no+// canonical order to mirror. Go randomises map iteration per range statement, so+// leaving it to the map would vary those three outputs even between two calls in+// one process; sorting gives a stable, machine-independent order instead.+func sortedKeptNames(names map[string]bool) []string {+ out := make([]string, 0, len(names))+ for n := range names {+ out = append(out, n)+ }+ sort.Strings(out)+ return out+}+// drillWithAncestors pulls the ancestors of names into the kept set so a path// renders as root -> ... -> node, mirroring _with_ancestors in _render_toc_drilldown.func drillWithAncestors(names map[string]bool, parents map[string]string) map[string]bool {+ // Ranging a map while adding keys visits the new ones in an unspecified order,+ // but the result is order-independent: every start walks up to the root (or the+ // depth guard), so the SET is the same whatever order the names are visited in+ // — the same closure Python builds from its `list(names)` snapshot (:1648).for name := range names {cur := parents[name]guard := 0@@ -1186,7 +1264,7 @@ func renderTocDrilldown(query string, qvec []float64, nodes []structureNode, rel}}keptNames = drillWithAncestors(names, parents)- for n := range keptNames {+ for _, n := range sortedKeptNames(keptNames) {if e, ok := byName[n]; ok {kept = append(kept, e)}@@ -1209,7 +1287,7 @@ func renderTocDrilldown(query string, qvec []float64, nodes []structureNode, rel}}keptNames = drillWithAncestors(names, parents)- for n := range keptNames {+ for _, n := range sortedKeptNames(keptNames) {if e, ok := byName[n]; ok {kept = append(kept, e)}@@ -1230,7 +1308,9 @@ func renderTocDrilldown(query string, qvec []float64, nodes []structureNode, relout.selector = selectorreturn out}- // depth_of: node -> number of kept ancestors (rendering indentation).+ // depth_of: node -> number of kept ancestors (rendering indentation). The+ // iteration order does not matter here: each entry is a pure function of+ // (node, parents, keptNames), so the map comes out the same either way.depthOf := map[string]int{}for n := range keptNames {d := 0diff --git a/internal/rag/advanced_rag/harness/tool_navigation_test.go b/internal/rag/advanced_rag/harness/tool_navigation_test.goindex 197fe23f9c..3fb7bdb4b0 100644--- a/internal/rag/advanced_rag/harness/tool_navigation_test.go+++ b/internal/rag/advanced_rag/harness/tool_navigation_test.go@@ -19,6 +19,7 @@ package harnessimport ("context""errors"+ "reflect""strings""testing")@@ -155,11 +156,108 @@ func TestNavigateTreeBadArgsAndInfra(t *testing.T) {if got := NavigateTree(context.Background(), nil, NavTreeInput{Query: "q"}); got.EmptyReason != ReasonInfra {t.Errorf("no router: reason = %q, want infra", got.EmptyReason)}+ // Every configured dataset failed to answer: infra, NOT no_structure. Route()+ // reports "no compiled tree" as (nil, nil), so a non-nil error is a failure —+ // a dataset-level verdict drawn from zero successful reads would mislabel an+ // outage, and no_structure is the verdict the session strikes a tool out on.got := NavigateTree(context.Background(), &stubRouter{err: errors.New("ES down")}, NavTreeInput{Query: "q", KbIDs: []string{"kb1"},})- if got.EmptyReason != ReasonNoStructure {- t.Errorf("backend error: reason = %q, want no_structure (no dataset verdict)", got.EmptyReason)+ if got.EmptyReason != ReasonInfra {+ t.Errorf("backend error: reason = %q, want infra", got.EmptyReason)+ }+ if !got.HasStructure() {+ t.Error("an outage must not read as a structure absence (HasStructure must stay true)")+ }+}++// perKBRouter answers each dataset differently, so a partially failed fan-out can+// be exercised; stubRouter answers the same for every dataset.+type perKBRouter struct {+ results map[string][][2]string+ errs map[string]error+}++func (r *perKBRouter) Route(_ context.Context, _, kbID, _ string, _ []string, _ int) ([][2]string, error) {+ if err := r.errs[kbID]; err != nil {+ return nil, err+ }+ return r.results[kbID], nil+}++// TestNavigateTreePartialFailureIsInfra pins the mixed fan-out: one dataset fails+// while another legitimately reports no compiled tree. The failed read leaves the+// datasets' state unknown, so the answer must stay infra — concluding+// no_structure there would end the tool on evidence that was never read.+func TestNavigateTreePartialFailureIsInfra(t *testing.T) {+ got := NavigateTree(context.Background(), &perKBRouter{+ results: map[string][][2]string{"kb2": nil}, // kb2 answered: no compiled tree+ errs: map[string]error{"kb1": errors.New("ES down")},+ }, NavTreeInput{Query: "q", KbIDs: []string{"kb1", "kb2"}})+ if got.EmptyReason != ReasonInfra {+ t.Errorf("partial failure: reason = %q, want infra", got.EmptyReason)+ }+ // A failed descent has no Python counterpart to copy — Python lets the+ // exception escape and navigation.py has no empty_reason for it — so this+ // label is the port's own wording; only the XML shape must be Python's.+ if want := "<tree_navigation count=\"0\" error=\"nav tree descent failed\">\n</tree_navigation>"; got.Text != want {+ t.Errorf("partial failure: text = %q, want %q", got.Text, want)+ }++ // A dataset that answered AND routed still wins: one failed read must not+ // discard the documents another dataset returned.+ got = NavigateTree(context.Background(), &perKBRouter{+ results: map[string][][2]string{"kb2": [][2]string{{"doc-a", "summary"}}},+ errs: map[string]error{"kb1": errors.New("ES down")},+ }, NavTreeInput{Query: "q", KbIDs: []string{"kb1", "kb2"}})+ if got.EmptyReason != "" || len(got.DocIDs) != 1 || got.DocIDs[0] != "doc-a" {+ t.Errorf("partial success: reason = %q, doc_ids = %v, want doc-a from kb2", got.EmptyReason, got.DocIDs)+ }+}++// TestNavigateTreeEmptyXMLMatchesPython pins the text each empty NavResult carries.+// Python builds it in place: `count="0"` plus an `error="..."` attribute that is+// OMITTED for a query-level miss (navigation.py:877 "no retriever", :880 "query is+// required", :899 no attribute). The model does not read it — both sides replace+// every empty_reason with their own note (action_session.py:869-876,+// tool_executor.go:402) — but the NavResult value must match.+func TestNavigateTreeEmptyXMLMatchesPython(t *testing.T) {+ cases := []struct {+ name string+ got NavResult+ want string+ }{+ {+ "bad_args",+ NavigateTree(context.Background(), &stubRouter{}, NavTreeInput{}),+ "<tree_navigation count=\"0\" error=\"query is required\">\n</tree_navigation>",+ },+ {+ "infra: no router",+ NavigateTree(context.Background(), nil, NavTreeInput{Query: "q"}),+ "<tree_navigation count=\"0\" error=\"no retriever\">\n</tree_navigation>",+ },+ {+ "no_doc: no attribute",+ NavigateTree(context.Background(), &stubRouter{emptyResult: true}, NavTreeInput{+ Query: "q", KbIDs: []string{"kb1"},+ }),+ "<tree_navigation count=\"0\">\n</tree_navigation>",+ },+ {+ // Go-only state: Python's nav-tree route has no no_structure verdict,+ // so this label is the port's own wording.+ "no_structure",+ NavigateTree(context.Background(), &stubRouter{nilResult: true}, NavTreeInput{+ Query: "q", KbIDs: []string{"kb1"},+ }),+ "<tree_navigation count=\"0\" error=\"no compiled navigation tree\">\n</tree_navigation>",+ },+ }+ for _, c := range cases {+ if c.got.Text != c.want {+ t.Errorf("%s: text = %q, want %q", c.name, c.got.Text, c.want)+ }}}@@ -652,6 +750,61 @@ func TestBuildTocTreeNoRelationsYieldsIsolatedRoots(t *testing.T) {}}+// TestStructureGraphFromRawMissingFieldsStayEmpty pins the compiled-payload+// defaults: a field a compiled row simply does not carry must read as absent, so+// the renderers apply Python's defaults ("other" for a type, "related_to" for a+// relation type) and a relation with a missing endpoint is DROPPED. fmt.Sprint(nil)+// returns the non-empty string "<nil>", which defeated all three: the model read+// "- Name (<nil>): <nil>" and the tree grew a phantom edge to "<nil>"+// (navigation.py:1714-1715, 1783-1788).+func TestStructureGraphFromRawMissingFieldsStayEmpty(t *testing.T) {+ rawEntities := []map[string]any{+ {"name": "Bare"}, // no type, no description+ {"name": "Typed", "type": "PERSON"}, // no description+ {"name": " ", "type": "PERSON"}, // nameless rows are dropped+ }+ rawRels := []map[string]any{+ {"from": "A", "to": "B"}, // no type+ {"from": "A"}, // missing endpoint -> dropped+ {"to": "B"}, // missing endpoint -> dropped+ {"from": "A", "to": "A"}, // self-loop -> dropped+ }+ nodes, rels := structureGraphFromRaw(rawEntities, rawRels)++ if len(nodes) != 2 {+ t.Fatalf("entities = %+v, want 2 (the nameless row dropped)", nodes)+ }+ if nodes[0].Type != "" || nodes[0].Description != "" {+ t.Errorf("missing fields = %q/%q, want empty so the renderer defaults the type to \"other\"",+ nodes[0].Type, nodes[0].Description)+ }+ if nodes[1].Type != "PERSON" || nodes[1].Description != "" {+ t.Errorf("entity = %+v, want PERSON with an empty description", nodes[1])+ }+ if len(rels) != 1 {+ t.Errorf("relations = %+v, want only the complete one", rels)+ }+ if rels[0].relType != "" {+ t.Errorf("relation type = %q, want empty so the renderer defaults it to \"related_to\"", rels[0].relType)+ }++ // End to end: the outline the model reads must show the Python defaults, not+ // "<nil>" and not the dropped relations.+ outline := renderOutline(structureNodesFromEntities(nodes), rels)+ if strings.Contains(outline, "<nil>") {+ t.Fatalf("outline leaks <nil>:\n%s", outline)+ }+ if !strings.Contains(outline, "- Bare (other)") {+ t.Errorf("outline = %q, want the missing type defaulted to \"other\"", outline)+ }+ if !strings.Contains(outline, "- A -[related_to]-> B") {+ t.Errorf("outline = %q, want the missing relation type defaulted to \"related_to\"", outline)+ }+ if got := strings.Count(outline, "->"); got != 1 {+ t.Errorf("outline has %d relation line(s), want 1 (missing endpoints and self-loops dropped):\n%s", got, outline)+ }+}+// TestStructureDocSegmentFormat locks the per-doc XML shape navigateStructures// renders for each routed/requested document, mirroring Python's// <doc rank doc_id doc_title="" entities relations> element (navigation.py@@ -810,6 +963,101 @@ func TestRenderTocDrilloutBeamFallback(t *testing.T) {}}+// stableDrillout runs call repeatedly and fails if any kept-node-derived output+// differs between runs (Go randomises map iteration per range statement).+func stableDrillout(t *testing.T, call func() structureDrillout) structureDrillout {+ t.Helper()+ first := call()+ for i := 1; i < 64; i++ {+ got := call()+ if got.outline != first.outline || got.chunkPtrs != first.chunkPtrs || got.nodes != first.nodes ||+ !reflect.DeepEqual(got.chunkPaths, first.chunkPaths) {+ t.Fatalf("run %d differs from run 0:\n--- run 0\n%s\n--- run %d\n%s",+ i, first.outline, i, got.outline)+ }+ }+ return first+}++// assertKeptNodeOrder requires the outline's node lines to appear in the sorted+// order the kept names are materialised in.+func assertKeptNodeOrder(t *testing.T, outline string) {+ t.Helper()+ prev := -1+ for _, name := range []string{"Alpha", "Beta", "Gamma", "Root"} {+ at := strings.Index(outline, "- "+name+" (")+ if at < 0 {+ continue // this strategy did not keep the node+ }+ if at < prev {+ t.Errorf("node line %q is out of sorted order:\n%s", name, outline)+ }+ prev = at+ }+}++// TestTocDrilldownKeptOrderIsDeterministic pins the kept-node ordering across all+// three selection strategies. kept drives the rendered outline, chunkPaths (first+// writer wins for a chunk covered by several nodes) and the capped chunk list+// whose snippets reach the model, so an arbitrary order hands back a different+// outline AND a different chunk subset on every run. Python iterates a kept_names+// SET (navigation.py:1660, 1673, 1594) — arbitrary, and hash-randomised per+// process — so there is no canonical order to mirror; the port sorts the names.+func TestTocDrilldownKeptOrderIsDeterministic(t *testing.T) {+ nodes := []structureNode{+ {name: "Root", nodeType: "tree_node", desc: "root node", sourceChunkIDs: []string{"r1", "shared"}},+ {name: "Beta", nodeType: "tree_node", desc: "beta node", sourceChunkIDs: []string{"b1", "b2"}},+ {name: "Alpha", nodeType: "tree_node", desc: "alpha node", sourceChunkIDs: []string{"a1", "a2"}},+ {name: "Gamma", nodeType: "tree_node", desc: "gamma node", sourceChunkIDs: []string{"g1", "shared"}},+ }+ rels := []structureRel{+ // Deliberately not the order the node names sort in.+ {from: "Root", to: "Beta"},+ {from: "Root", to: "Alpha"},+ {from: "Root", to: "Gamma"},+ }++ t.Run("llm_toc", func(t *testing.T) {+ // The model's picks arrive in ITS order; it must not leak into the output.+ got := stableDrillout(t, func() structureDrillout {+ return renderTocDrilldown("q", nil, nodes, rels, drillLoader, nil, []string{"Gamma", "Alpha", "Beta"})+ })+ assertKeptNodeOrder(t, got.outline)+ // chunkPaths follows the same order: "shared" is covered by both Gamma and+ // Root, and sorted order visits Gamma first, so Gamma's path wins.+ want := map[string]string{+ "a1": "Root -> Alpha", "a2": "Root -> Alpha",+ "b1": "Root -> Beta", "b2": "Root -> Beta",+ "g1": "Root -> Gamma", "shared": "Root -> Gamma",+ "r1": "Root",+ }+ if !reflect.DeepEqual(got.chunkPaths, want) {+ t.Errorf("chunkPaths = %v, want %v", got.chunkPaths, want)+ }+ // Snippets are capped at structMaxChunks in kept order, so "r1" (Root sorts+ // last) sits outside a stable, repeatable window.+ if !strings.Contains(got.outline, "[chunk g1]") || strings.Contains(got.outline, "[chunk r1]") {+ t.Errorf("outline's snippet window is not the first %d chunks in kept order:\n%s",+ structMaxChunks, got.outline)+ }+ })++ t.Run("chunk_retrieval", func(t *testing.T) {+ got := stableDrillout(t, func() structureDrillout {+ return renderTocDrilldown("q", nil, nodes, rels, drillLoader,+ []chunkHit{{id: "g1", score: 0.9}, {id: "a1", score: 0.7}}, nil)+ })+ assertKeptNodeOrder(t, got.outline)+ })++ t.Run("beam", func(t *testing.T) {+ got := stableDrillout(t, func() structureDrillout {+ return renderTocDrilldown("alpha beta gamma root", nil, nodes, rels, drillLoader, nil, nil)+ })+ assertKeptNodeOrder(t, got.outline)+ })+}+func TestParseCompiledStructureCarriesVec(t *testing.T) {rows := []StructureRow{{CompileKwd: "tree",diff --git a/internal/rag/advanced_rag/harness/tool_search.go b/internal/rag/advanced_rag/harness/tool_search.goindex e0743ee8d1..ce68acaabc 100644--- a/internal/rag/advanced_rag/harness/tool_search.go+++ b/internal/rag/advanced_rag/harness/tool_search.go@@ -45,7 +45,10 @@ const (DefaultTopN = 12DefaultRerankCandidatesCount = 64DefaultTopK = 1024- // maxEffectiveQueryChars caps the expanded query (Python: [:400]).+ // maxEffectiveQueryChars caps the expanded query in CODE POINTS: Python+ // slices a str with `[:400]` (search.py:129/216/254), which counts code+ // points. A byte cap would cut a multi-byte rune in half and, for CJK, would+ // apply a ~133-character cap instead of 400.maxEffectiveQueryChars = 400// maxQueryTerms caps the terms derived from a query (Python// _query_to_terms: [:16]).@@ -167,14 +170,20 @@ type DocTenantResolver interface {// 0.0 keyword-only (BM25). This is how Python's three search entry points// (hybrid_search / vector_search / bm25_search) differ.type RetrieveRequest struct {- Query string- DatasetIDs []string- DocScope []string- TopN int- TopK int- RerankCandidatesCount int- SimilarityThreshold float64- KeywordsSimilarityWeight float64+ Query string+ DatasetIDs []string+ DocScope []string+ TopN int+ TopK int+ RerankCandidatesCount int+ // SimilarityThreshold and KeywordsSimilarityWeight are pointers because ZERO+ // is a valid explicit setting (threshold 0 = no floor; weight 0 = the vector+ // leg gets the whole weight). nil means "not supplied", so the retriever+ // keeps its own default instead of being handed a zero override.+ SimilarityThreshold *float64+ // KeywordsSimilarityWeight is named for keywords but carries the VECTOR+ // weight, as in Python's vector_similarity_weight.+ KeywordsSimilarityWeight *float64TenantID string// MetaDataFilter restricts retrieval to chunks whose metadata matches// (Python tools.meta_data_filter). Nil means no filtering.@@ -637,10 +646,12 @@ func runSearch(ctx context.Context, deps SearchDeps, p SearchParams, opts search// `f"{query} {keywords}".strip()` / `query` forms, which carry no [:400].var effectiveQuery stringif strings.TrimSpace(p.RetrievalQuery) != "" {- effectiveQuery = strings.TrimSpace(fmt.Sprintf("%s %s", p.Question, p.RetrievalQuery))- if len(effectiveQuery) > maxEffectiveQueryChars {- effectiveQuery = effectiveQuery[:maxEffectiveQueryChars]- }+ // Python's cap slices code points (`f"{query} {retrieval_query}".strip()[:400]`,+ // search.py:129/216/254): a byte slice would both cut a multi-byte rune in+ // half — handing the retriever invalid UTF-8 — and, for CJK, stop at+ // ~133 characters, silently dropping two thirds of the expansion terms+ // the fan-out leg weighs on.+ effectiveQuery = truncateRunes(strings.TrimSpace(fmt.Sprintf("%s %s", p.Question, p.RetrievalQuery)), maxEffectiveQueryChars)} else if strings.TrimSpace(p.Keywords) != "" {effectiveQuery = strings.TrimSpace(fmt.Sprintf("%s %s", p.Question, p.Keywords))} else {@@ -696,14 +707,14 @@ func runSearch(ctx context.Context, deps SearchDeps, p SearchParams, opts searchTopN: topN,TopK: intOrDef(deps.TopK, DefaultTopK),RerankCandidatesCount: max(intOrDef(deps.RerankCandidatesCount, DefaultRerankCandidatesCount), topN),- SimilarityThreshold: opts.threshold,+ SimilarityThreshold: &opts.threshold,// The field is named for keywords but carries the VECTOR weight, as in// Python's vector_similarity_weight. Whether the vector leg runs at all// — and its default — is decided by the calling entry-point function// (see searchOpts.weight), NOT by a shared channel flag. ExcludeCompiled// mirrors Python hybrid_search's must_not={"exists":"compile_kwd"}// (search.py:hybrid_search), which keeps compiled products out of plain retrieval.- KeywordsSimilarityWeight: opts.weight,+ KeywordsSimilarityWeight: &opts.weight,TenantID: deps.TenantID,MetaDataFilter: deps.MetaDataFilter,RankFeature: rankFeature,@@ -1000,33 +1011,45 @@ func WebSearchTool(ctx context.Context, deps SearchDeps, args map[string]any) (Tvar evidenceIDs []stringnewChunks := 0seen := make(map[string]bool, len(results))- for i, r := range results {- if r == "" || seen[r] {- continue- }- seen[r] = true- chunkID := fmt.Sprintf("web_%d", i)- c := map[string]any{- "chunk_id": chunkID,- "content": r,- "doc_id": "web",- }- if deps.KB != nil {- before := len(deps.KB.Chunks)- deps.KB.Merge([]map[string]any{c}, nil)- newChunks += len(deps.KB.Chunks) - before- }- // Evidence references the chunk id (Python ids.append(cid)), not a pool- // position.- evidenceIDs = append(evidenceIDs, chunkID)- // Passage shape from _admit_evidence(include_doc_id=False): {"id","content"},- // non-table text cut to 1200 code points (plain slice, no ellipsis).- content := r- if !IsTableChunk(c) {- content = truncateRunes(content, 1200)+ // The whole batch is ONE critical section (Kbinfos.Admit): Python's admit loop+ // has no await (the retrieval itself is awaited above it), so two sessions can+ // never interleave here.+ deps.KB.Admit(func(p *PoolAdmitter) {+ for i, r := range results {+ if r == "" || seen[r] {+ continue+ }+ // Python _admit_evidence early-stops at the pool cap, BEFORE it records+ // the chunk as seen, so a rejected passage is retried once room frees.+ if p.Full() {+ continue+ }+ seen[r] = true+ chunkID := fmt.Sprintf("web_%d", i)+ c := map[string]any{+ "chunk_id": chunkID,+ "content": r,+ "doc_id": "web",+ }+ // p.Add, not Merge: Merge takes this same pool lock and would deadlock+ // inside the critical section. It also answers "new to the pool" per+ // chunk, so the count no longer has to be inferred from a length delta+ // that other sessions' appends inflate.+ if p.Add(c) {+ newChunks+++ }+ // Evidence references the chunk id (Python ids.append(cid)), not a pool+ // position.+ evidenceIDs = append(evidenceIDs, chunkID)+ // Passage shape from _admit_evidence(include_doc_id=False): {"id","content"},+ // non-table text cut to 1200 code points (plain slice, no ellipsis).+ content := r+ if !IsTableChunk(c) {+ content = truncateRunes(content, 1200)+ }+ payload = append(payload, map[string]any{"id": chunkID, "content": content})}- payload = append(payload, map[string]any{"id": chunkID, "content": content})- }+ })if len(payload) == 0 {return ToolOutcome{Payload: []any{}, Status: StatusMiss, Reason: ReasonNoDoc, Metrics: map[string]any{"hits": 0, "new_evidence": 0}}, nil@@ -1112,35 +1135,43 @@ func (e *searchExecutor) listChunks(ctx context.Context, args map[string]any) (Tif len(admit) > listChunksMaxOut {admit = admit[:listChunksMaxOut]}- kbSeen := make(map[string]bool, len(e.deps.KB.Chunks))- for _, c := range e.deps.KB.Chunks {- kbSeen[chunkKey(c)] = true- }seen := map[string]bool{}var payload []anyvar evidenceIDs []stringnewChunks := 0- for _, c := range admit {- cid := ChunkIDOf(c)- if cid == "" || seen[cid] {- continue- }- seen[cid] = true- evidenceIDs = append(evidenceIDs, cid)- // Shape like _admit_evidence(include_doc_id=False): {"id","content"} with- // non-table text cut to 1200 code points (a plain slice, no ellipsis).- content := ChunkTextOf(c)- if !IsTableChunk(c) {- content = truncateRunes(content, 1200)- }- payload = append(payload, map[string]any{"id": cid, "content": content})- key := chunkKey(c)- if !kbSeen[key] {- kbSeen[key] = true- e.deps.KB.Chunks = append(e.deps.KB.Chunks, c)- newChunks+++ // The batch is ONE critical section (Kbinfos.Admit): Python's admit stretch+ // has no await, so two sessions can never interleave here, and the pool-side+ // dedup must see the LIVE pool rather than a snapshot taken upfront.+ e.deps.KB.Admit(func(p *PoolAdmitter) {+ for _, c := range admit {+ cid := ChunkIDOf(c)+ if cid == "" {+ // Python _exec_list_chunks skips a blank cid in the caller, BEFORE+ // _admit_evidence.+ continue+ }+ // Python _admit_evidence early-stops at the pool cap, BEFORE the+ // per-call dedup.+ if p.Full() {+ continue+ }+ if seen[cid] {+ continue+ }+ seen[cid] = true+ evidenceIDs = append(evidenceIDs, cid)+ // Shape like _admit_evidence(include_doc_id=False): {"id","content"} with+ // non-table text cut to 1200 code points (a plain slice, no ellipsis).+ content := ChunkTextOf(c)+ if !IsTableChunk(c) {+ content = truncateRunes(content, 1200)+ }+ payload = append(payload, map[string]any{"id": cid, "content": content})+ if p.Add(c) {+ newChunks+++ }}- }+ })if len(payload) == 0 {return ToolOutcome{Payload: []any{}, Status: StatusMiss, Reason: ReasonNoDoc, Metrics: map[string]any{"hits": 0, "new_evidence": 0}}, nildiff --git a/internal/rag/advanced_rag/harness/tool_search_test.go b/internal/rag/advanced_rag/harness/tool_search_test.goindex d81a98a12f..ca8ec959bf 100644--- a/internal/rag/advanced_rag/harness/tool_search_test.go+++ b/internal/rag/advanced_rag/harness/tool_search_test.go@@ -27,6 +27,7 @@ import ("reflect""strings""testing"+ "unicode/utf8""ragflow/internal/entity")@@ -107,6 +108,43 @@ func TestHybridSearchBuildsEffectiveQuery(t *testing.T) {}}+// TestHybridSearchEffectiveQueryCapsCodePoints pins the expanded-query cap to+// code points, not bytes. Python slices a str with `[:400]` (search.py:129/216/254),+// so the cap counts code points: a byte slice both splits a multi-byte rune —+// handing the retriever invalid UTF-8 — and caps a CJK query at ~133 characters,+// dropping expansion terms the fan-out leg weighs on. The ASCII case above+// cannot tell the two apart (bytes == runes there).+func TestHybridSearchEffectiveQueryCapsCodePoints(t *testing.T) {+ // "who made it" + " " is 12 bytes, so byte 400 lands one byte inside a CJK+ // rune (400-12 = 388 = 3*129 + 1): the byte slice is not even valid UTF-8.+ expansion := strings.Repeat("知识", 300) // 600 runes, 1800 bytes+ r := &stubRetriever{chunks: []map[string]any{{"content": "hit"}}}+ deps, _ := newTestSearchDeps(r)+ HybridSearch(context.Background(), deps, SearchParams{+ Question: "who made it", RetrievalQuery: expansion,+ })++ if len(r.requests) != 1 {+ t.Fatalf("requests = %d, want 1", len(r.requests))+ }+ q := r.requests[0].Query+ if !utf8.ValidString(q) {+ t.Fatalf("query is not valid UTF-8 — a byte slice cut a rune in half: %q", q)+ }+ if got := utf8.RuneCountInString(q); got != maxEffectiveQueryChars {+ t.Errorf("query runes = %d, want the cap at %d code points (%d bytes)", got, maxEffectiveQueryChars, len(q))+ }+ full := "who made it " + expansion+ if want := string([]rune(full)[:maxEffectiveQueryChars]); q != want {+ t.Errorf("query = %q, want the first %d code points %q", q, maxEffectiveQueryChars, want)+ }+ // 400 code points of mostly-CJK text is far more than 400 bytes: the cap is a+ // character budget, not a byte budget.+ if len(q) <= maxEffectiveQueryChars {+ t.Errorf("query bytes = %d, want > %d (the cap counts code points)", len(q), maxEffectiveQueryChars)+ }+}+func TestHybridSearchPassesRankFeature(t *testing.T) {// label_question -> rank_feature must flow into the underlying retrieve// request (Python retrieve: rank_feature=label_question(question, self.kbs),@@ -193,7 +231,7 @@ func TestHybridSearchStoresMemoryBeforeNarrowing(t *testing.T) {if MemorySize(kb) != 1 {t.Fatalf("memory size = %d, want 1", MemorySize(kb))}- if strings.Contains(ChunkTextOf(kb.Memory[0]), "<em>") {+ if strings.Contains(ChunkTextOf(kb.Memory[0]), "*Culdcept*") {t.Error("memory must hold the RAW chunk, not the narrowed/highlighted copy")}}@@ -241,7 +279,7 @@ func TestNarrowContentKeepsNeighboursAndHighlights(t *testing.T) {if !ok {t.Fatal("NarrowContent must match")}- if !strings.Contains(got, "<em>Culdcept</em>") {+ if !strings.Contains(got, "*Culdcept*") {t.Errorf("keyword not highlighted: %q", got)}// +/-1 neighbour window: Alpha and Omega should be present.@@ -275,14 +313,90 @@ func TestSplitKeywordsFallsBackToBigrams(t *testing.T) {func TestHighlightKeywordsPrefersLongestTerm(t *testing.T) {// "new york" must win over "york" so the shorter term cannot split it.got := HighlightKeywords("welcome to New York city", []string{"york", "new york"})- if !strings.Contains(got, "<em>New York</em>") {+ // Python's star marker, and ONE span for the multi-word entity+ // (text_processing.py:391-394) — never "*New* *York*".+ if !strings.Contains(got, "*New York*") {t.Errorf("highlight = %q, want the longest term applied", got)}- if strings.Contains(got, "<em>york</em>") {+ if strings.Contains(got, "*york*") {t.Errorf("shorter term must not win: %q", got)}}+// TestHighlightKeywordsFoldsWithoutByteOffsets pins the matching to rune space.+// `strings.ToLower` is not byte-length-preserving: "İ" (U+0130) is 2 bytes and+// folds to the 1-byte "i" (Go applies the simple 1:1 case mapping, the opposite+// direction from Python's full fold, which expands it to 3 bytes). So a byte+// offset taken from the original indexes the folded string at a different+// position: the loop then runs past its end (panic: slice bounds out of range+// [10:9]) or cuts a rune in half (invalid UTF-8). Corpus text reaches this via+// NarrowContent, e.g. a Turkish document.+func TestHighlightKeywordsFoldsWithoutByteOffsets(t *testing.T) {+ // "İstanbul" is 9 bytes but folds to the 8-byte "istanbul", so the old byte+ // loop wrote text[0:8] — one byte short, ending mid-word ("İstanbu").+ if got := HighlightKeywords("İstanbul is here", []string{"istanbul"}); got != "*İstanbul* is here" {+ t.Errorf("highlight = %q, want the whole word wrapped", got)+ }++ // Two shrunken runes push the byte index one past the end of the folded+ // string while the original still has a byte left: the old loop panicked.+ got := HighlightKeywords("İİstanbul", []string{"istanbul"})+ if !utf8.ValidString(got) {+ t.Fatalf("highlight is not valid UTF-8 — a span split a rune: %q", got)+ }+ if !strings.Contains(got, "*İstanbul*") {+ t.Errorf("highlight = %q, want the fold-matched runes wrapped in place", got)+ }+}++// TestHighlightKeywordsFoldsUppercaseKeywords pins that a keyword's OWN casing is+// folded the way the haystack is: terms are matched against the lowercased text+// (`lows`), and Python builds its phrase list with `(kw or "").strip().lower()`+// (text_processing.py:396) plus re.IGNORECASE (:411). A caller-supplied "Rocket"+// used to be compared verbatim, so a capitalised keyword never matched and the+// span was silently left unstarred.+func TestHighlightKeywordsFoldsUppercaseKeywords(t *testing.T) {+ for _, tc := range []struct {+ name string+ text string+ kwds []string+ want string+ }{+ {"single word", "the Rocket launched", []string{"Rocket"}, "the *Rocket* launched"},+ {"multi-word phrase stays one span", "welcome to New York city", []string{"New York"}, "welcome to *New York* city"},+ {"source casing is preserved", "ROCKET is loud", []string{"Rocket"}, "*ROCKET* is loud"},+ {"surrounding spaces are trimmed", "the Rocket launched", []string{" Rocket "}, "the *Rocket* launched"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ if got := HighlightKeywords(tc.text, tc.kwds); got != tc.want {+ t.Errorf("HighlightKeywords(%q, %v) = %q, want %q", tc.text, tc.kwds, got, tc.want)+ }+ })+ }+}++// TestHighlightKeywordsKeepsPhrasePartsWhole pins Python's phrase guard+// (text_processing.py:405): a stem-matched word that already occurs inside a+// keyword phrase is NOT added as a term of its own, so a standalone "Braves" is+// left alone and only the "Atlanta Braves" span is starred.+func TestHighlightKeywordsKeepsPhrasePartsWhole(t *testing.T) {+ got := HighlightKeywords("Braves lost. Atlanta Braves won.", []string{"Atlanta Braves"})+ if want := "Braves lost. *Atlanta Braves* won."; got != want {+ t.Errorf("highlight = %q, want %q", got, want)+ }+}++// TestHighlightKeywordsStemMatchesCapitalisedWords pins the word scan: Python+// stems every `[A-Za-z]+` word (:403), so a capitalised inflected word still+// contributes its stem term. The shared lowercase pattern matched only the+// fragment after the capital ("Nominated" -> "ominated"), so nothing was starred.+func TestHighlightKeywordsStemMatchesCapitalisedWords(t *testing.T) {+ got := HighlightKeywords("Nominated twice.", []string{"nominations"})+ if want := "*Nominated* twice."; got != want {+ t.Errorf("highlight = %q, want %q", got, want)+ }+}+// ---------------------------------------------------------------------------// Doc aggregations// ---------------------------------------------------------------------------@@ -391,6 +505,17 @@ func (r *stubRetriever) lastReq(t *testing.T) RetrieveRequest {return r.requests[len(r.requests)-1]}+// ptrFloat dereferences a control pointer. nil means "the caller supplied+// nothing, so the retriever keeps its own default"; the search-leg tests below+// assert the opposite (an explicit value is passed), so nil is a failure here.+func ptrFloat(t *testing.T, p *float64) float64 {+ t.Helper()+ if p == nil {+ t.Fatal("control pointer = nil, want an explicit value")+ }+ return *p+}+// TestVectorSearchBailsWithoutEmbedder mirrors Python vector_search: with no// embedder configured it returns nothing (Python bails when embd_mdl is unset),// and must NOT even hit the backend.@@ -413,8 +538,8 @@ func TestVectorSearchWeightIsOne(t *testing.T) {deps.HasEmbedder = trueVectorSearch(context.Background(), deps, SearchParams{Question: "q"})req := r.lastReq(t)- if req.KeywordsSimilarityWeight != 1.0 {- t.Errorf("vector search weight = %v, want 1.0", req.KeywordsSimilarityWeight)+ if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != 1.0 {+ t.Errorf("vector search weight = %v, want 1.0", got)}if !req.ExcludeCompiled {t.Error("vector search must exclude compiled rows")@@ -428,11 +553,11 @@ func TestBM25SearchUsesZeroWeight(t *testing.T) {deps, _ := newTestSearchDeps(r)BM25Search(context.Background(), deps, SearchParams{Question: "q"})req := r.lastReq(t)- if req.KeywordsSimilarityWeight != 0 {- t.Errorf("bm25 search weight = %v, want 0", req.KeywordsSimilarityWeight)+ if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != 0 {+ t.Errorf("bm25 search weight = %v, want 0", got)}- if req.SimilarityThreshold != 0 {- t.Errorf("bm25 search threshold = %v, want 0", req.SimilarityThreshold)+ if got := ptrFloat(t, req.SimilarityThreshold); got != 0 {+ t.Errorf("bm25 search threshold = %v, want 0", got)}if !req.ExcludeCompiled {t.Error("bm25 search must exclude compiled rows")@@ -446,8 +571,8 @@ func TestGrepSearchDelegatesToBM25(t *testing.T) {deps, _ := newTestSearchDeps(r)GrepSearch(context.Background(), deps, SearchParams{Question: "q", Keywords: "kw"})req := r.lastReq(t)- if req.KeywordsSimilarityWeight != 0 {- t.Errorf("grep search weight = %v, want 0", req.KeywordsSimilarityWeight)+ if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != 0 {+ t.Errorf("grep search weight = %v, want 0", got)}if !req.ExcludeCompiled {t.Error("grep search must exclude compiled rows")@@ -567,7 +692,7 @@ func TestGrepSearchDerivesKeywordsHint(t *testing.T) {if want := "who made Culdcept? who made Culdcept"; req.Query != want {t.Errorf("query = %q, want %q (question + derived terms)", req.Query, want)}- if req.KeywordsSimilarityWeight != 0 || !req.ExcludeCompiled {+ if ptrFloat(t, req.KeywordsSimilarityWeight) != 0 || !req.ExcludeCompiled {t.Error("grep must stay keyword-only (weight 0) and exclude compiled rows")}// The derived hint also drives the narrowing stage: a prose candidate whose@@ -622,8 +747,8 @@ func TestHybridSearchExcludesCompiledAndWeightsThreeTenths(t *testing.T) {deps.HasEmbedder = trueHybridSearch(context.Background(), deps, SearchParams{Question: "q"})req := r.lastReq(t)- if req.KeywordsSimilarityWeight != HybridSearchDefaultVectorWeight {- t.Errorf("hybrid search weight = %v, want %v", req.KeywordsSimilarityWeight, HybridSearchDefaultVectorWeight)+ if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != HybridSearchDefaultVectorWeight {+ t.Errorf("hybrid search weight = %v, want %v", got, HybridSearchDefaultVectorWeight)}if !req.ExcludeCompiled {t.Error("hybrid search must exclude compiled rows")@@ -642,8 +767,8 @@ func TestRetrieveSearchDoesNotExcludeCompiled(t *testing.T) {if req.ExcludeCompiled {t.Error("retrieve search must NOT exclude compiled rows")}- if req.KeywordsSimilarityWeight != DefaultHybridVectorWeight {- t.Errorf("retrieve search weight = %v, want %v", req.KeywordsSimilarityWeight, DefaultHybridVectorWeight)+ if got := ptrFloat(t, req.KeywordsSimilarityWeight); got != DefaultHybridVectorWeight {+ t.Errorf("retrieve search weight = %v, want %v", got, DefaultHybridVectorWeight)}}diff --git a/internal/rag/advanced_rag/harness/tool_text_processing.go b/internal/rag/advanced_rag/harness/tool_text_processing.goindex 7438cff69e..8ae6ba1ce4 100644--- a/internal/rag/advanced_rag/harness/tool_text_processing.go+++ b/internal/rag/advanced_rag/harness/tool_text_processing.go@@ -150,6 +150,24 @@ func SplitKeywords(keywords string) []string {var wordRe = regexp.MustCompile("[a-z0-9]+")+// wordLetterRe is the scan Python's _highlight_keywords uses to find the words+// to stem-match: `re.findall(r"[A-Za-z]+", text)` (text_processing.py:403). It is+// deliberately NOT the shared lowercase wordRe, which on capitalized text matches+// only fragments ("New" -> "ew", "Nominated" -> "ominated") and therefore never+// yields the stem term Python adds for those words.+var wordLetterRe = regexp.MustCompile("[A-Za-z]+")++// containedInPhrase reports whether low occurs inside any keyword phrase,+// mirroring Python's `any(low in p for p in phrases)` (text_processing.py:405).+func containedInPhrase(low string, phrases map[string]struct{}) bool {+ for p := range phrases {+ if strings.Contains(p, low) {+ return true+ }+ }+ return false+}+// stemSuffixes mirrors Python _STEM_SUFFIXES (longest-first; first match wins).var stemSuffixes = [][2]string{{"ations", ""}, {"ation", ""}, {"ated", ""}, {"ates", ""}, {"ate", ""},@@ -332,13 +350,26 @@ func NarrowContent(content string, kwds []string) (string, bool) {return "..." + HighlightKeywords(b.String(), kwds) + "...", true}-// HighlightKeywords wraps keyword occurrences in <em>, longest term first so a-// longer keyword is not partially consumed by a shorter one.+// HighlightKeywords stars keyword occurrences, longest term first so a longer+// keyword is not partially consumed by a shorter one. The marker is a STAR, not+// an XML tag: Python returns `*term*` (text_processing.py:412) and its docstring+// relies on it — a multi-word entity must stay ONE contiguous span+// ("*Atlanta Braves*", never "*Atlanta* *Braves*") for the downstream+// entity cross-check. The <em> tags elsewhere in this port are the ENGINE's+// highlight markup (rag/utils/*_conn.py, agentic_search.go), a different layer.func HighlightKeywords(text string, kwds []string) string {if len(kwds) == 0 {return text}terms := append([]string(nil), kwds...)+ // Python's phrase set (text_processing.py:396): keywords trimmed, lowercased+ // and deduplicated. It guards the stem terms added just below.+ phrases := make(map[string]struct{}, len(kwds))+ for _, kw := range kwds {+ if p := strings.ToLower(strings.TrimSpace(kw)); p != "" {+ phrases[p] = struct{}{}+ }+ }// Stem-based highlight terms: a stemmed form that matches in the text is// wrapped too, mirroring Python _highlight_keywords (which adds stemmed-word// occurrences so e.g. "nominated" is highlighted for keyword "nominations").@@ -350,41 +381,85 @@ func HighlightKeywords(text string, kwds []string) string {stemSet[s] = true}}- for _, word := range wordRe.FindAllString(text, -1) {+ for _, word := range wordLetterRe.FindAllString(text, -1) {low := strings.ToLower(word)- if stemmable(low) && stemSet[stem(low)] {+ // Python adds a stem-matched word only when it is NOT already inside a+ // keyword phrase (text_processing.py:405): "nominated" is starred for+ // "nominations", while the "Braves" of "Atlanta Braves" is left to the+ // phrase's own span instead of being starred on its own elsewhere.+ if stemmable(low) && stemSet[stem(low)] && !containedInPhrase(low, phrases) {terms = append(terms, low)}}}- sort.SliceStable(terms, func(i, j int) bool { return len(terms[i]) > len(terms[j]) })+ // Match in RUNE space. `strings.ToLower` is not byte-length-preserving: "İ"+ // is 2 bytes and folds to the 1-byte "i", so a byte offset taken from the+ // original indexes the folded string at a different position. The loop then+ // slices past the end of the folded string (panic: slice bounds out of+ // range) or cuts a rune in half and emits invalid UTF-8. Go's case mapping is+ // 1:1 per RUNE, so a rune index is valid in both strings. (Python is immune+ // for a different reason: re.sub re-emits m.group(0) from the ORIGINAL text+ // instead of re-slicing it — text_processing.py:411-412.)+ rs := []rune(text)+ lows := []rune(strings.ToLower(text))+ if len(lows) != len(rs) {+ // Unreachable while the fold stays rune-for-rune; kept so a future switch+ // to a full case fold (which does change the rune count) degrades to+ // plain text instead of misaligned spans.+ return text+ }+ // Longest term first, compared by rune count: Python sorts on str length,+ // i.e. code points (text_processing.py:396/411).+ termRunes := make([][]rune, 0, len(terms))+ for _, t := range terms {+ // Fold the TERM the same way the haystack was folded: the match below+ // compares against `lows`, and only the stem-derived terms appended above+ // were already lowercase — a caller-supplied "Rocket" or "New York" kept+ // its casing and therefore never matched, silently dropping the highlight.+ // Python builds its phrase list with `(kw or "").strip().lower()`+ // (text_processing.py:396) and additionally compiles with re.IGNORECASE+ // (:411), so the trim and the case fold both belong here.+ if t = strings.ToLower(strings.TrimSpace(t)); t != "" {+ termRunes = append(termRunes, []rune(t))+ }+ }+ sort.SliceStable(termRunes, func(i, j int) bool { return len(termRunes[i]) > len(termRunes[j]) })var b strings.Builder- low := strings.ToLower(text)- i := 0- for i < len(text) {- best := -1+ for i := 0; i < len(rs); {bestLen := 0- for _, t := range terms {- if t == "" {- continue- }- if strings.HasPrefix(low[i:], t) && len(t) > bestLen {- best, bestLen = i, len(t)+ for _, t := range termRunes {+ if len(t) > bestLen && runesHavePrefix(lows[i:], t) {+ bestLen = len(t)}}- if best < 0 {- b.WriteByte(text[i])+ if bestLen == 0 {+ b.WriteRune(rs[i])i++continue}- b.WriteString("<em>")- b.WriteString(text[i : i+bestLen])- b.WriteString("</em>")+ // Emit the ORIGINAL runes, so the highlight keeps the source casing+ // (Python re-emits m.group(0)), wrapped in Python's star marker.+ b.WriteString("*")+ b.WriteString(string(rs[i : i+bestLen]))+ b.WriteString("*")i += bestLen}return b.String()}+// runesHavePrefix reports whether hay starts with needle.+func runesHavePrefix(hay, needle []rune) bool {+ if len(needle) == 0 || len(needle) > len(hay) {+ return false+ }+ for i, r := range needle {+ if hay[i] != r {+ return false+ }+ }+ return true+}+// IsFactDenseSentence reports whether a sentence carries a fact-bearing signal: a// number / year / percentage / magnitude word (_FACT_RE) or a proper noun that is// not part of an abbreviation run. Mirrors Python _is_fact_dense_sentencediff --git a/internal/rag/prompts/action_run.md b/internal/rag/prompts/action_run.mdindex 696b966e08..059cf50cc2 100644--- a/internal/rag/prompts/action_run.md+++ b/internal/rag/prompts/action_run.md@@ -36,3 +36,36 @@ CRITICAL RULES- ACTION COMPLETION IS MANDATORY: when you have what you need (or hit a dead end), output the state patch now. Do not ask to continue searching.- Unverifiable candidates must be eliminated (set candidate null) with a clue documenting why.- Partial verification is OK: record a candidate at tentative strength (0.4-0.7) if you can't fully verify it yet, and move on.++# TOOL PLAYBOOK++You get tools only in medium / high (7 tools: `retrieve`, `search_chunks`, `list_chunks`, `navigate_tree`, `navigate_structure`, `calculate`, `web_search`) and ultra (those 7 + `graph_explore`). Low mode has NO tool loop — answer with plain retrieval. Every native tool call still REQUIRES the decision envelope from CRITICAL RULES (it is a mandatory tool parameter, not optional).++## 1. Combination chains (call in this order)++- **You already hold a `doc_id`** → `navigate_structure(doc_id, query)` to find the right passage, then `list_chunks(doc_id)` to read it. Do NOT call `navigate_tree` first.+- **No `doc_id` yet, and the corpus is large** → `navigate_tree(query)` to route to candidate documents, take a `doc_id`, then `navigate_structure(doc_id, query)` → `list_chunks(doc_id)`.+- **Exact term / short answer** → `retrieve(query[1-3])` first; if snippets are insufficient, `search_chunks(query[1-2])` (semantic, may find passages with NO shared surface words); if you need the full document, `list_chunks(doc_id)`.+- **You must DERIVE a number** → first collect every needed number with any of the above, then `calculate(question, facts)` with the facts verbatim, and report the computed result as-is. If the answer is already one of the stated numbers, answer directly.+- **Relational multi-hop (ultra only)** → get a start entity from `search_chunks` / `navigate_structure`, then `graph_explore(query, doc_scope)`.++## 2. Convergence rules (hard, enforced by the runtime — follow them to avoid wasted turns)++- Make at most **1-2 useful tool calls per direction**, then emit a state patch. Do not keep searching once the direction is reasonably exhausted.+- Re-submitting the SAME intent with a paraphrase is intercepted as a near-duplicate and SKIPPED (you get a nudge, not new results). Change the angle or patch what you have.+- If a compile-only tool (`navigate_tree` / `navigate_structure` / `graph_explore`) returns "no compiled structure", switch to `search_chunks` / `retrieve` / `list_chunks` **immediately**. A second such result disables that tool for the REST of the session — do not retry it.+- `web_search` only appears when a web provider is configured; if it does, use it ONLY for world knowledge / time-sensitive facts that plausibly live outside the fixed corpus.+- `list_chunks` accepts ONLY `doc_id` (no `chunk_ids` argument) — you cannot ask it to read specific chunks; it returns the whole document (capped).++## 3. What a tool result means → your next action++Each tool returns a status. Act on it:++| Status | Meaning | Your next action |+| --- | --- | --- |+| `ok` | New evidence entered the shared pool | Fill slots / move to the next direction |+| `miss` | This query matched nothing, but the tool itself is valid | Rephrase or switch tools — do NOT conclude the dataset lacks it |+| `empty` (`no_structure`) | Dataset-level: no such compiled structure exists | Switch to `search_chunks` / `retrieve` / `list_chunks` now; retrying disables the tool |+| `poor` | Output returned but too weak to use | Add evidence with another tool |+| `redundant` | Every hit was already in your evidence | Stop re-searching; emit a `<state>` patch with what you have |+| `error` | Infrastructure / provider failure | Switch tools; do not retry the same call |diff --git a/internal/rag/prompts/generator.go b/internal/rag/prompts/generator.goindex f965a1bfe3..2b1a9e4c00 100644--- a/internal/rag/prompts/generator.go+++ b/internal/rag/prompts/generator.go@@ -38,13 +38,9 @@ import ("strings""ragflow/internal/common"+ "ragflow/internal/tokenizer")-// approxCharsPerToken approximates token cost of a string (see-// tokenizer.NumTokensFromString) so the evidence budget can be enforced without-// a tokenizer dependency.-const approxCharsPerToken = 4-var _LOG = common.StdLogger()var reNewlines = regexp.MustCompile(`\n+`)@@ -139,61 +135,92 @@ func CitationPrompt(userDefined string) string {return strings.TrimSpace(string(data)) + citationIDSuffix}+// kbpBlock renders one knowledge block (id / title / url / metadata / content)+// for the given 1-based index. ok is false when the chunk carries no content,+// which KBPrompt treats as "skip" (mirrors Python's `if not c: continue`).+//+// The block starts with a newline, mirroring Python's `"\nID: {}".format(...)`,+// so a caller that joins the blocks with "\n" renders the same prompt Python+// does.+func kbpBlock(c map[string]any, index int) (string, bool) {+ content := chunkText(c)+ if strings.TrimSpace(content) == "" {+ return "", false+ }+ block := fmt.Sprintf("\nID: %d", index)+ if title := chunkTitle(c); title != "" {+ block += "\n├── Title: " + flattenNewlines(title)+ }+ if url, _ := c["url"].(string); url != "" {+ block += "\n├── URL: " + flattenNewlines(url)+ }+ if meta, ok := c["document_metadata"].(map[string]any); ok {+ keys := make([]string, 0, len(meta))+ for k := range meta {+ keys = append(keys, k)+ }+ sort.Strings(keys)+ for _, k := range keys {+ if mv := meta[k]; mv != nil {+ if v := flattenNewlines(fmt.Sprint(mv)); v != "" {+ block += "\n├── " + k + ": " + v+ }+ }+ }+ }+ block += "\n└── Content:\n" + content+ return block, true+}+// KBPrompt mirrors rag/prompts/generator.kb_prompt: render chunks into the// numbered knowledge blocks the citation rules refer to.//// Blocks are numbered by position (1-based) so the model's [n] citations match// the block order it sees, matching Python's `ID: {i}` rendering (hash_id=False).//+// The budget is applied to the COMPLETE rendered block (title / url / metadata /+// content), measured in TOKENS with the same cl100k_base encoder Python's+// num_tokens_from_string uses: budgeting only the content in a char approximation+// left the decoration unaccounted, so the rendered prompt could exceed the budget+// it claims to enforce.+//// maxTokens must be positive: Python's caller applies its own budget// (min(chat_mdl.max_length, _EVIDENCE_BUDGET_TOKENS)) before calling.func KBPrompt(chunks []map[string]any, maxTokens int) []string {- budgetChars := int(float64(maxTokens) * 0.97 * approxCharsPerToken)+ blocks, _ := KBPromptWithSourceIndices(chunks, maxTokens)+ return blocks+}- var selected []map[string]any+// KBPromptWithSourceIndices is KBPrompt plus, for every rendered block, the+// index in chunks it was rendered from.+//+// Callers that need "the blocks of these chunks" must use the indices instead of+// assuming one block per chunk: a chunk with no content is skipped and the loop+// stops at the token budget, so len(blocks) <= len(chunks) and blocks[i] is not+// chunks[i]. Slicing by a chunk count can therefore drop readable blocks or+// point past the end.+func KBPromptWithSourceIndices(chunks []map[string]any, maxTokens int) ([]string, []int) {+ out := make([]string, 0, len(chunks))+ sources := make([]int, 0, len(chunks))used := 0- for _, c := range chunks {+ for idx, c := range chunks {if c == nil {continue}- content := chunkText(c)- if strings.TrimSpace(content) == "" {+ block, ok := kbpBlock(c, len(out)+1)+ if !ok {continue}- n := len(content)- if budgetChars < used+n {- _LOG.Printf("[KBPrompt] Not all the retrieval into prompt: %d/%d", len(selected), len(chunks))+ n := tokenizer.NumTokensFromString(block)+ if float64(maxTokens)*0.97 < float64(used+n) {+ _LOG.Printf("[KBPrompt] Not all the retrieval into prompt: %d/%d", len(out), len(chunks))break}used += n- selected = append(selected, c)- }-- out := make([]string, 0, len(selected))- for i, c := range selected {- block := fmt.Sprintf("ID: %d", i+1)- if title := chunkTitle(c); title != "" {- block += "\n├── Title: " + flattenNewlines(title)- }- if url, _ := c["url"].(string); url != "" {- block += "\n├── URL: " + flattenNewlines(url)- }- if meta, ok := c["document_metadata"].(map[string]any); ok {- keys := make([]string, 0, len(meta))- for k := range meta {- keys = append(keys, k)- }- sort.Strings(keys)- for _, k := range keys {- if v := flattenNewlines(fmt.Sprint(meta[k])); v != "" {- block += "\n├── " + k + ": " + v- }- }- }- block += "\n└── Content:\n" + chunkText(c)out = append(out, block)+ sources = append(sources, idx)}- return out+ return out, sources}// flattenNewlines mirrors Python kb_prompt's `re.sub(r"\n+", " ", line)`.@@ -204,10 +231,18 @@ func flattenNewlines(s string) string {// chunkText mirrors Python kb_prompt's// get_value(ck, "content", "content_with_weight").func chunkText(c map[string]any) string {- if v, ok := GetValue(c, "content", "content_with_weight").(string); ok {- return v+ v := GetValue(c, "content", "content_with_weight")+ if v == nil {+ // Neither content field is present: return empty so KBPrompt skips the+ // chunk (Python's `if not c: continue`). fmt.Sprint(nil) would yield the+ // literal "<nil>", which would both render into the prompt and consume+ // the token budget.+ return ""+ }+ if s, ok := v.(string); ok {+ return s}- return fmt.Sprint(GetValue(c, "content", "content_with_weight"))+ return fmt.Sprint(v)}// chunkTitle mirrors Python kb_prompt'sdiff --git a/internal/rag/prompts/generator_test.go b/internal/rag/prompts/generator_test.goindex 46c1c40d64..40d813effe 100644--- a/internal/rag/prompts/generator_test.go+++ b/internal/rag/prompts/generator_test.go@@ -20,6 +20,8 @@ import ("reflect""strings""testing"++ "ragflow/internal/tokenizer")func TestGetValue(t *testing.T) {@@ -142,3 +144,101 @@ func TestCitationPromptHonoursOverride(t *testing.T) {t.Errorf("CitationPrompt(override) = %q, want the override verbatim", got)}}++// TestKBPromptSkipsChunksWithoutContent pins Python's `if not c: continue`: a+// chunk whose content fields are absent (or explicitly null) must be skipped,+// not rendered as "\n└── Content:\n<nil>".+func TestKBPromptSkipsChunksWithoutContent(t *testing.T) {+ chunks := []map[string]any{+ {"chunk_id": "a"}, // neither content field+ {"chunk_id": "b", "content": nil}, // explicit null+ {"chunk_id": "c", "content_with_weight": "hi"}, // the only usable chunk+ }+ got := KBPrompt(chunks, 1000)+ if len(got) != 1 {+ t.Fatalf("blocks = %d, want 1 (content-less chunks must be skipped): %v", len(got), got)+ }+ if strings.Contains(got[0], "<nil>") {+ t.Errorf("block rendered a nil field: %q", got[0])+ }+ if !strings.Contains(got[0], "hi") {+ t.Errorf("block lost its content: %q", got[0])+ }+}++// TestKBPromptSkipsNilMetadataValues pins Python draw_node's `if not line: return+// ""` for None: a null metadata value must not render as "<nil>".+func TestKBPromptSkipsNilMetadataValues(t *testing.T) {+ got := KBPrompt([]map[string]any{{+ "content": "body",+ "document_metadata": map[string]any{"author": nil, "page": "3"},+ }}, 1000)+ if len(got) != 1 {+ t.Fatalf("blocks = %d, want 1", len(got))+ }+ if strings.Contains(got[0], "<nil>") {+ t.Errorf("nil metadata rendered literally: %q", got[0])+ }+ if !strings.Contains(got[0], "page: 3") {+ t.Errorf("non-nil metadata lost: %q", got[0])+ }+}++// TestKBPromptBudgetsCompleteBlock pins that the budget covers the title / url /+// metadata decoration, not just the content: a tiny-content chunk whose block is+// dominated by metadata must be dropped when it cannot fit the token budget.+func TestKBPromptBudgetsCompleteBlock(t *testing.T) {+ lean := map[string]any{"content": "hi"}+ fat := map[string]any{+ "content": "hi",+ "document_metadata": map[string]any{"blob": strings.Repeat("x", 500)},+ }+ leanBlock, _ := kbpBlock(lean, 1)+ fatBlock, _ := kbpBlock(fat, 1)+ leanTokens := tokenizer.NumTokensFromString(leanBlock)+ fatTokens := tokenizer.NumTokensFromString(fatBlock)+ if leanTokens == 0 || fatTokens <= leanTokens {+ t.Fatalf("cl100k tokenizer unavailable: lean=%d fat=%d tokens", leanTokens, fatTokens)+ }+ // A budget that just fits the lean block must reject the metadata-heavy one.+ maxTokens := int(float64(leanTokens)/0.97) + 1+ if got := KBPrompt([]map[string]any{fat}, maxTokens); len(got) != 0 {+ t.Errorf("metadata-heavy block (%d tokens) must not fit a ~%d-token budget; got %d block(s)",+ fatTokens, maxTokens, len(got))+ }+ if got := KBPrompt([]map[string]any{lean}, maxTokens); len(got) != 1 {+ t.Errorf("lean block (%d tokens) should fit a ~%d-token budget; got %d block(s)",+ leanTokens, maxTokens, len(got))+ }+}++// TestKBPromptWithSourceIndicesTracksRenderedChunks locks the contract callers+// need to slice blocks safely: every rendered block reports the index of the+// chunk it came from, and a chunk with no content contributes no block. A caller+// that slices by a chunk count instead would shift by the skipped chunks.+func TestKBPromptWithSourceIndicesTracksRenderedChunks(t *testing.T) {+ chunks := []map[string]any{+ {"chunk_id": "c0", "content": ""}, // skipped: no content+ {"chunk_id": "c1", "content": "first body"},+ nil, // skipped: not a chunk+ {"chunk_id": "c2", "content": "second body"},+ }++ blocks, sources := KBPromptWithSourceIndices(chunks, 100000)+ if len(blocks) != 2 {+ t.Fatalf("blocks = %d, want 2 (only the chunks with content)", len(blocks))+ }+ want := []int{1, 3}+ if !reflect.DeepEqual(sources, want) {+ t.Fatalf("sources = %v, want %v", sources, want)+ }+ for i, src := range sources {+ if got := chunks[src]["chunk_id"]; got != nil && !strings.Contains(blocks[i], "Content:") {+ t.Errorf("block %d (source %v) is not a rendered chunk: %q", i, got, blocks[i])+ }+ }+ // KBPrompt keeps the same rendering, just without the source indices.+ if plain := KBPrompt(chunks, 100000); !reflect.DeepEqual(plain, blocks) {+ t.Fatalf("KBPrompt = %v, want the same blocks as KBPromptWithSourceIndices", plain)+ }+}diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.goindex f67c95415a..e81ce7788a 100644--- a/internal/service/chat_pipeline.go+++ b/internal/service/chat_pipeline.go@@ -732,14 +732,6 @@ func (s *ChatPipelineService) AsyncChat(})}}- // The harness streams think-then-answer inside ONE compose call;- // if it never reached a non-think delta (reasoning-only output)- // close the block so no consumer sees an unpaired <think>.- defer func() {- if harnessThinking {- sink("", false)- }- }()// Python dialog_service.py:2077 — the web provider is handed to// RAGTools only when the internet flag enables web search;// otherwise web_search stays off the agentic tool surface.@@ -748,6 +740,13 @@ func (s *ChatPipelineService) AsyncChat(webSearch = s.harnessWebSearchFunc(chat.PromptConfig)}hk, harnessAnswer, hErr := s.retrieveViaHarness(ctx, question, kbs, docIDs, imageFiles, attachments, thinkingMode, chat.TenantID, chat.LLMID, chat.ID, webSearch, sink)+ // The harness streams think-then-answer inside ONE compose call.+ // Close the block here, once that call (and its trailing+ // narration line) has returned: a reasoning-only run would+ // otherwise keep it open until the goroutine exits, emitting+ // EndToThink after the Phase 10/11 answer or the Final below.+ // No-op when no block is open.+ sink("", false)if hErr != nil {common.Warn("harness retrieval failed", zap.Error(hErr))} else {@@ -3134,10 +3133,10 @@ func (s *ChatPipelineService) decorateHarnessAnswer(answer string, kbinfos map[sref[k] = v}if cRaw, ok := ref["chunks"].([]map[string]interface{}); ok {- for _, cm := range cRaw {- delete(cm, "vector")- }- ref["chunks"] = cRaw+ // chunksFormat builds the client-facing shape (content, document_name,+ // dataset_id, ...) in NEW maps, so the engine keys and the per-chunk+ // vector never reach the reference and the shared chunks stay intact.+ ref["chunks"] = chunksFormat(cRaw)}refs = ref}diff --git a/internal/service/chat_pipeline_test.go b/internal/service/chat_pipeline_test.goindex 41946abded..fe34071638 100644--- a/internal/service/chat_pipeline_test.go+++ b/internal/service/chat_pipeline_test.go@@ -1643,3 +1643,43 @@ func TestRetrieveViaHarnessNaiveSkipsToolLoop(t *testing.T) {func TestToolLoopLineNilSink(t *testing.T) {toolLoopLine(nil, "[Tool loop] Step 1: running rag...") // must not panic}++// TestDecorateHarnessAnswerReferenceUsesClientChunkShape pins that the reasoning+// path hands consumers the same reference shape as the non-reasoning one+// (content / document_name / dataset_id — not the engine keys), and that the+// source chunks are left untouched (no in-place vector stripping).+func TestDecorateHarnessAnswerReferenceUsesClientChunkShape(t *testing.T) {+ src := map[string]interface{}{+ "chunk_id": "c1",+ "content_with_weight": "hello",+ "docnm_kwd": "Doc One",+ "doc_id": "d1",+ "kb_id": "kb1",+ "vector": []float64{0.1, 0.2},+ }+ kbinfos := map[string]interface{}{+ "chunks": []map[string]interface{}{src},+ "doc_aggs": []interface{}{map[string]interface{}{"doc_id": "d1", "doc_name": "Doc One"}},+ }++ s := &ChatPipelineService{}+ res := s.decorateHarnessAnswer("The answer [ID:0]", kbinfos)+ if res.Reference == nil {+ t.Fatal("a cited answer must carry a reference")+ }+ chunks, _ := res.Reference["chunks"].([]map[string]interface{})+ if len(chunks) != 1 {+ t.Fatalf("reference chunks = %#v, want 1", res.Reference["chunks"])+ }+ if chunks[0]["content"] != "hello" || chunks[0]["document_name"] != "Doc One" || chunks[0]["dataset_id"] != "kb1" {+ t.Errorf("reference chunk = %#v, want the client-facing keys populated", chunks[0])+ }+ for _, engineKey := range []string{"content_with_weight", "docnm_kwd", "kb_id", "vector"} {+ if _, has := chunks[0][engineKey]; has {+ t.Errorf("reference chunk leaked engine key %q", engineKey)+ }+ }+ if _, has := src["vector"]; !has {+ t.Error("the shared source chunk lost its vector: the reference must not mutate it")+ }+}diff --git a/internal/service/model_service.go b/internal/service/model_service.goindex 6eb3c4b31e..89659ab1df 100644--- a/internal/service/model_service.go+++ b/internal/service/model_service.go@@ -3553,21 +3553,12 @@ func (m *ModelProviderService) GetModelConfigByID(ctx context.Context, userID streturn nil, "", nil, 0, fmt.Errorf("provider id=%s not found for model id=%s", modelEntity.ProviderID, modelID)}- if providerEntity.TenantID != userID {- userTenants, terr := NewUserTenantService().GetUserTenantRelationByUserIDWithContext(ctx, userID)- if terr != nil {- return nil, "", nil, 0, terr- }- allowed := false- for _, rel := range userTenants {- if rel != nil && rel.TenantID == providerEntity.TenantID {- allowed = true- break- }- }- if !allowed {- return nil, "", nil, 0, fmt.Errorf("tenant %s has no access to provider owned by tenant %s", userID, providerEntity.TenantID)- }+ allowed, accessErr := m.tenantCanReachProviderTenant(ctx, userID, providerEntity.TenantID)+ if accessErr != nil {+ return nil, "", nil, 0, accessErr+ }+ if !allowed {+ return nil, "", nil, 0, fmt.Errorf("tenant %s has no access to provider owned by tenant %s", userID, providerEntity.TenantID)}instanceEntity, err := m.modelInstanceDAO.GetByID(ctx, dao.DB, modelEntity.InstanceID)@@ -3668,37 +3659,122 @@ func (m *ModelProviderService) ResolveModelContextLength(ctx context.Context, te// support must skip the outer rag_agent react loop and fall back to the direct// graph (Python falls back to async_chat).//-// Precedence mirrors Python's tenant_model_service.get_model_config_by_id-// (:363) `"is_tools": model_extra.get("is_tools", is_tool)`: the flag persisted+// Precedence mirrors Python's tenant_model_service+// `"is_tools": model_extra.get("is_tools", is_tool)` (:363): the flag persisted// on the tenant model wins, and the provider catalog is only a default for-// models enrolled without one. Reading the catalog first instead would send a-// tenant-disabled model through the outer react loop — where a model that does-// not actually call tools answers from its own knowledge and the retrieval-// never runs.+// models enrolled without one. That is why the tenant_model row is resolved+// first — by UUID, or for a composite "model@instance@provider" reference+// through the tenant's own provider/instance rows. Reading the catalog first+// instead would send a tenant-disabled model through the outer react loop —+// where a model that does not actually call tools answers from its own+// knowledge and the retrieval never runs.+//+// The reference is validated before model_extra is read, mirroring Python's+// get_model_config_by_id (:324-347): a missing or disabled model, a model not+// enrolled as this type, a missing provider, or a provider the tenant cannot+// reach returns an error rather than a definitive is_tools answer. Callers treat+// an error as "no tool support", the same outcome as an unset flag.func (m *ModelProviderService) ResolveModelToolSupport(ctx context.Context, tenantID string, modelType entity.ModelType, modelRef string) (bool, error) {if strings.TrimSpace(modelRef) == "" {return false, fmt.Errorf("model ref is required")}- // Tenant-model UUID path: the persisted extra flag first, then the catalog.++ // Tenant-model UUID path. IDs are globally unique and may belong to a+ // provider shared with a joined tenant, so the lookup itself is unscoped —+ // tenantCanReachProviderTenant below decides access.if modelObj, err := m.modelDAO.GetByID(ctx, dao.DB, modelRef); err == nil {- if ts, ok := extraToolSupport(modelObj.Extra); ok {- return ts, nil+ if modelObj.Status != "active" {+ return false, fmt.Errorf("tenant model id=%s is disabled", modelRef)}- if prov, perr := m.modelProviderDAO.GetByID(ctx, dao.DB, modelObj.ProviderID); perr == nil && prov != nil {- return catalogToolSupport(prov.ProviderName, modelObj.ModelName), nil+ if !entity.ModelType(modelObj.ModelType).Has(modelType) {+ return false, fmt.Errorf("tenant model id=%s cannot be used as %s model", modelRef, modelType.String())}- return false, nil+ provider, provErr := m.modelProviderDAO.GetByID(ctx, dao.DB, modelObj.ProviderID)+ if provErr != nil && !errors.Is(provErr, gorm.ErrRecordNotFound) {+ return false, provErr+ }+ if provider == nil {+ return false, fmt.Errorf("provider id=%s not found for model id=%s", modelObj.ProviderID, modelRef)+ }+ allowed, accessErr := m.tenantCanReachProviderTenant(ctx, tenantID, provider.TenantID)+ if accessErr != nil {+ return false, accessErr+ }+ if !allowed {+ return false, fmt.Errorf("tenant %s has no access to provider owned by tenant %s", tenantID, provider.TenantID)+ }+ return toolSupportFromTenantModel(modelObj.Extra, provider.ProviderName, modelObj.ModelName), nil} else if !errors.Is(err, gorm.ErrRecordNotFound) {return false, err}- // Composite "model@instance@provider" path: parse and consult the catalog.- pureModelName, _, providerName, err := parseModelName(modelRef)++ // Composite "model@instance@provider" path. The tenant's own rows are+ // resolved so a flag persisted on the enrolled model still beats the+ // catalog, exactly as it does on the UUID path.+ pureModelName, instanceName, providerName, err := parseModelName(modelRef)if err != nil {return false, err}+ provider, err := m.modelProviderDAO.GetByTenantIDAndProviderName(ctx, dao.DB, tenantID, providerName)+ if err != nil {+ return false, fmt.Errorf("provider %q lookup failed: %w", providerName, err)+ }+ if provider == nil {+ return false, fmt.Errorf("provider %q not found for model %q", providerName, modelRef)+ }+ instance, err := m.modelInstanceDAO.GetByProviderIDAndInstanceName(ctx, dao.DB, provider.ID, instanceName)+ if err != nil {+ return false, fmt.Errorf("instance %q lookup failed: %w", instanceName, err)+ }+ if instance == nil {+ return false, fmt.Errorf("instance %q not found for model %q", instanceName, modelRef)+ }+ modelObj, err := m.modelDAO.GetByProviderIDAndInstanceIDAndModelTypeAndModelName(ctx, dao.DB, provider.ID, instance.ID, int(modelType), pureModelName)+ if err == nil {+ if modelObj.Status != "active" {+ return false, fmt.Errorf("model %q is disabled", modelRef)+ }+ return toolSupportFromTenantModel(modelObj.Extra, provider.ProviderName, modelObj.ModelName), nil+ }+ if !errors.Is(err, gorm.ErrRecordNotFound) {+ return false, fmt.Errorf("model %q lookup failed: %w", modelRef, err)+ }++ // The tenant never enrolled this model as this type: the catalog is the only+ // source left (the instance api_key payload Python reads here is seeded from+ // the same catalog at enrolment time).return catalogToolSupport(providerName, pureModelName), nil}+// toolSupportFromTenantModel returns the is_tools flag persisted on a tenant+// model's extra JSON when present, otherwise the provider catalog's declaration+// for that model. Mirrors Python's model_extra.get("is_tools", is_tool).+func toolSupportFromTenantModel(extra, providerName, modelName string) bool {+ if ts, ok := extraToolSupport(extra); ok {+ return ts+ }+ return catalogToolSupport(providerName, modelName)+}++// tenantCanReachProviderTenant reports whether userID owns the provider's tenant+// or is a joined member of it. Mirrors Python's tenant_model_service+// get_model_config_by_id tenant check (:342-347).+func (m *ModelProviderService) tenantCanReachProviderTenant(ctx context.Context, userID, ownerTenantID string) (bool, error) {+ if userID == ownerTenantID {+ return true, nil+ }+ userTenants, err := NewUserTenantService().GetUserTenantRelationByUserIDWithContext(ctx, userID)+ if err != nil {+ return false, err+ }+ for _, rel := range userTenants {+ if rel != nil && rel.TenantID == ownerTenantID {+ return true, nil+ }+ }+ return false, nil+}+// extraToolSupport reads the is_tools flag persisted on a tenant model's extra// JSON. The value is written as a JSON boolean (addModelToInstance stores// llm.Tools.Support verbatim) but has historically also been spelled as adiff --git a/internal/service/model_service_test.go b/internal/service/model_service_test.goindex a99d29123e..74c2a43f66 100644--- a/internal/service/model_service_test.go+++ b/internal/service/model_service_test.go@@ -942,3 +942,180 @@ func TestSplitRightAnchoredModelName(t *testing.T) {})}}++// TestModelProviderServiceResolveModelToolSupportValidatesTenantModelRefs pins+// the validation Python's get_model_config_by_id performs before model_extra is+// read (tenant_model_service.py:324-347): a disabled model, a model not enrolled+// as the requested type, and a model whose provider the caller's tenant cannot+// reach must not produce a definitive tool-support answer.+func TestModelProviderServiceResolveModelToolSupportValidatesTenantModelRefs(t *testing.T) {+ db := setupModelProviderServiceTestDB(t)+ useModelProviderServiceTestDB(t, db)+ activeStatus := "1"+ rows := []interface{}{+ &entity.UserTenant{ID: "ut-1", UserID: "tenant-1", TenantID: "tenant-1", Role: "owner", InvitedBy: "tenant-1", Status: &activeStatus},+ &entity.TenantModelProvider{ID: "provider-own", TenantID: "tenant-1", ProviderName: "OpenAI"},+ &entity.TenantModelProvider{ID: "provider-other", TenantID: "tenant-2", ProviderName: "Anthropic"},+ &entity.TenantModelInstance{ID: "instance-own", ProviderID: "provider-own", InstanceName: "default", APIKey: "sk-test", Status: "active", Extra: "{}"},+ &entity.TenantModelInstance{ID: "instance-other", ProviderID: "provider-other", InstanceName: "default", APIKey: "sk-test", Status: "active", Extra: "{}"},+ &entity.TenantModel{ID: "model-active", ProviderID: "provider-own", InstanceID: "instance-own", ModelName: "gpt-test", ModelType: int(entity.ModelTypeChat), Status: "active", Extra: `{"is_tools":true}`},+ &entity.TenantModel{ID: "model-disabled", ProviderID: "provider-own", InstanceID: "instance-own", ModelName: "gpt-test-off", ModelType: int(entity.ModelTypeChat), Status: "inactive", Extra: `{"is_tools":true}`},+ &entity.TenantModel{ID: "model-embedding", ProviderID: "provider-own", InstanceID: "instance-own", ModelName: "text-embedding-test", ModelType: int(entity.ModelTypeEmbedding), Status: "active", Extra: `{"is_tools":true}`},+ &entity.TenantModel{ID: "model-foreign", ProviderID: "provider-other", InstanceID: "instance-other", ModelName: "claude-test", ModelType: int(entity.ModelTypeChat), Status: "active", Extra: `{"is_tools":true}`},+ }+ for _, row := range rows {+ if err := db.Create(row).Error; err != nil {+ t.Fatalf("failed to seed %T: %v", row, err)+ }+ }++ svc := NewModelProviderService()+ ctx := t.Context()++ // A reference that passes validation still reads the persisted flag.+ got, err := svc.ResolveModelToolSupport(ctx, "tenant-1", entity.ModelTypeChat, "model-active")+ if err != nil || !got {+ t.Fatalf("active chat model = (%v, %v), want (true, nil)", got, err)+ }++ for _, tc := range []struct {+ name string+ modelID string+ }{+ {"disabled", "model-disabled"},+ {"not enrolled as chat", "model-embedding"},+ {"provider owned by another tenant", "model-foreign"},+ } {+ got, err := svc.ResolveModelToolSupport(ctx, "tenant-1", entity.ModelTypeChat, tc.modelID)+ if err == nil {+ t.Errorf("%s: err = nil, want a validation error", tc.name)+ }+ if got {+ t.Errorf("%s: tool support = true, want false", tc.name)+ }+ }+}++// TestModelProviderServiceResolveModelToolSupportPrefersTenantFlag covers the+// composite "<model>@<instance>@<provider>" reference shape (chat.llm_id and the+// harness ModelID accept both a UUID and a composite ref): the flag persisted on+// the tenant_model row must beat the provider catalog in both directions,+// mirroring Python's model_extra.get("is_tools", is_tool), and a disabled row+// must be rejected instead of falling back to the catalog.+func TestModelProviderServiceResolveModelToolSupportPrefersTenantFlag(t *testing.T) {+ catalog := dao.GetModelProviderManager().FindProvider("OpenAI")+ if catalog == nil {+ t.Skip("OpenAI catalog is unavailable")+ }+ var toolsOn, toolsOff string+ for _, m := range catalog.Models {+ if strings.Contains(m.Name, "@") {+ continue // the composite ref parser would split such a name+ }+ if toolsOn == "" && m.Tools != nil && m.Tools.Support {+ toolsOn = m.Name+ }+ if toolsOff == "" && (m.Tools == nil || !m.Tools.Support) {+ toolsOff = m.Name+ }+ }+ if toolsOn == "" || toolsOff == "" {+ t.Skipf("OpenAI catalog lacks both a tool-capable and a tool-incapable model (on=%q off=%q)", toolsOn, toolsOff)+ }+ if !catalogToolSupport("OpenAI", toolsOn) || catalogToolSupport("OpenAI", toolsOff) {+ t.Fatalf("catalog baseline changed: on=%v off=%v", catalogToolSupport("OpenAI", toolsOn), catalogToolSupport("OpenAI", toolsOff))+ }++ db := setupModelProviderServiceTestDB(t)+ useModelProviderServiceTestDB(t, db)+ rows := []interface{}{+ &entity.TenantModelProvider{ID: "provider-openai", TenantID: "tenant-1", ProviderName: "OpenAI"},+ &entity.TenantModelInstance{ID: "instance-openai", ProviderID: "provider-openai", InstanceName: "default", APIKey: "sk-test", Status: "active", Extra: "{}"},+ // Catalog says tools are supported; the tenant disabled them.+ &entity.TenantModel{ID: "model-on-off", ProviderID: "provider-openai", InstanceID: "instance-openai", ModelName: toolsOn, ModelType: int(entity.ModelTypeChat), Status: "active", Extra: `{"is_tools":false}`},+ // Catalog does not declare tools; the tenant enabled them.+ &entity.TenantModel{ID: "model-off-on", ProviderID: "provider-openai", InstanceID: "instance-openai", ModelName: toolsOff, ModelType: int(entity.ModelTypeChat), Status: "active", Extra: `{"is_tools":true}`},+ // Disabled row: must be rejected rather than resolved from the catalog.+ &entity.TenantModel{ID: "model-on-disabled", ProviderID: "provider-openai", InstanceID: "instance-openai", ModelName: toolsOn + "-disabled", ModelType: int(entity.ModelTypeChat), Status: "inactive", Extra: `{"is_tools":true}`},+ }+ for _, row := range rows {+ if err := db.Create(row).Error; err != nil {+ t.Fatalf("failed to seed %T: %v", row, err)+ }+ }++ svc := NewModelProviderService()+ ctx := t.Context()++ got, err := svc.ResolveModelToolSupport(ctx, "tenant-1", entity.ModelTypeChat, toolsOn+"@default@OpenAI")+ if err != nil {+ t.Fatalf("tool-capable model: err = %v", err)+ }+ if got {+ t.Errorf("tool support = true, want false: the tenant_model flag must win over the catalog")+ }++ got, err = svc.ResolveModelToolSupport(ctx, "tenant-1", entity.ModelTypeChat, toolsOff+"@default@OpenAI")+ if err != nil {+ t.Fatalf("tool-incapable model: err = %v", err)+ }+ if !got {+ t.Errorf("tool support = false, want true: the tenant_model flag must win over the catalog")+ }++ got, err = svc.ResolveModelToolSupport(ctx, "tenant-1", entity.ModelTypeChat, toolsOn+"-disabled@default@OpenAI")+ if err == nil || got {+ t.Errorf("disabled model = (%v, %v), want (false, error)", got, err)+ }+}++// TestModelProviderServiceResolveModelToolSupportPropagatesLookupFailure pins the+// distinction between an intentional not-found fallback and a real lookup+// failure: a database error must surface as an error instead of falling through+// to a successful (false, nil) answer, which callers read as "this model has no+// tool support".+func TestModelProviderServiceResolveModelToolSupportPropagatesLookupFailure(t *testing.T) {+ db := setupModelProviderServiceTestDB(t)+ useModelProviderServiceTestDB(t, db)+ activeStatus := "1"+ rows := []interface{}{+ &entity.UserTenant{ID: "ut-1", UserID: "tenant-1", TenantID: "tenant-1", Role: "owner", InvitedBy: "tenant-1", Status: &activeStatus},+ &entity.TenantModelProvider{ID: "provider-1", TenantID: "tenant-1", ProviderName: "OpenAI"},+ &entity.TenantModelInstance{ID: "instance-1", ProviderID: "provider-1", InstanceName: "default", APIKey: "sk-test", Status: "active", Extra: "{}"},+ &entity.TenantModel{ID: "model-1", ProviderID: "provider-1", InstanceID: "instance-1", ModelName: "gpt-test", ModelType: int(entity.ModelTypeChat), Status: "active", Extra: `{"is_tools":true}`},+ }+ for _, row := range rows {+ if err := db.Create(row).Error; err != nil {+ t.Fatalf("failed to seed %T: %v", row, err)+ }+ }++ svc := NewModelProviderService()+ ctx := t.Context()+ if got, err := svc.ResolveModelToolSupport(ctx, "tenant-1", entity.ModelTypeChat, "model-1"); err != nil || !got {+ t.Fatalf("baseline = (%v, %v), want (true, nil)", got, err)+ }++ // Drop the provider table: the lookup now fails with a database error+ // ("no such table"), which is neither a not-found fallback nor a+ // tool-support verdict.+ if err := db.Migrator().DropTable(&entity.TenantModelProvider{}); err != nil {+ t.Fatalf("failed to drop provider table: %v", err)+ }+ // Composite refs resolve the provider row directly.+ if got, err := svc.ResolveModelToolSupport(ctx, "tenant-1", entity.ModelTypeChat, "gpt-test@default@OpenAI"); err == nil {+ t.Errorf("composite provider lookup failure = (%v, nil), want a propagated error", got)+ }+ // The UUID path too.+ if got, err := svc.ResolveModelToolSupport(ctx, "tenant-1", entity.ModelTypeChat, "model-1"); err == nil {+ t.Errorf("uuid provider lookup failure = (%v, nil), want a propagated error", got)+ }++ // A failing tenant_model lookup must propagate as well.+ if err := db.Migrator().DropTable(&entity.TenantModel{}); err != nil {+ t.Fatalf("failed to drop model table: %v", err)+ }+ if got, err := svc.ResolveModelToolSupport(ctx, "tenant-1", entity.ModelTypeChat, "model-1"); err == nil {+ t.Errorf("model lookup failure = (%v, nil), want a propagated error", got)+ }+}diff --git a/internal/service/nav/chunk_agg_router.go b/internal/service/nav/chunk_agg_router.goindex af84810335..090fe79a2b 100644--- a/internal/service/nav/chunk_agg_router.go+++ b/internal/service/nav/chunk_agg_router.go@@ -119,7 +119,13 @@ func rankedAggregates(agg map[string]*chunkAggregate) []*chunkAggregate {// agentic harness stays free of the nlp/service layer; production wiring// supplies a hybrid retriever that excludes compiled rows (compile_kwd) — see// the navigation_tree retrieval contract.-type ChunkRetriever func(ctx context.Context, tenantID, kbID, query string, docScope []string, topN int, vecWeight float64) []map[string]any+//+// The error is part of the contract: a retrieval FAILURE must not be reported as+// an empty route. Python's _search_layers_nav_chunk_agg returns+// ok=False/SERVER_ERROR on a retrieval exception and reserves ok=True/total=0+// for "the pool aggregated to nothing", and the orchestrator picks its fallback+// from exactly that distinction.+type ChunkRetriever func(ctx context.Context, tenantID, kbID, query string, docScope []string, topN int, vecWeight float64) ([]map[string]any, error)// DocSummarizer loads the nav_doc "description" (the document's overall summary)// for a set of doc_ids, mirroring Python _nav_doc_summaries. Injected for the@@ -137,8 +143,10 @@ type ChunkAggRouter struct {// Route implements the agentic NavTreeRouter contract. It returns (nil, nil)// when no retriever is installed (treat as "no compiled tree", mirroring-// NavServiceRouter's contract), and an empty non-nil slice when a structure-// exists but no chunk routed to a doc.+// NavServiceRouter's contract), an empty non-nil slice when a structure exists+// but no chunk routed to a doc, and a non-nil error when retrieval itself failed+// — mirroring NavServiceRouter, which propagates a backend failure instead of+// reporting it as an empty route.func (r *ChunkAggRouter) Route(ctx context.Context, tenantID, kbID, query string, docScope []string, topK int) ([][2]string, error) {if r.Retrieve == nil {return nil, nil@@ -152,7 +160,12 @@ func (r *ChunkAggRouter) Route(ctx context.Context, tenantID, kbID, query string}// Pull a wide pool, then roll up per document (a focused doc may surface on// many chunks, so a small pool starves the aggregation).- chunks := r.Retrieve(ctx, tenantID, kbID, query, docScope, chunkAggPool, chunkAggVecWeight)+ chunks, err := r.Retrieve(ctx, tenantID, kbID, query, docScope, chunkAggPool, chunkAggVecWeight)+ if err != nil {+ // A retrieval failure is NOT "no document routed": the caller must be+ // able to take its fallback instead of re-phrasing the query.+ return nil, err+ }agg := aggregateChunks(chunks)if len(agg) == 0 {return make([][2]string, 0), nil // structure exists, nothing routeddiff --git a/internal/service/nav/chunk_agg_router_test.go b/internal/service/nav/chunk_agg_router_test.goindex e8a57390fa..69969e6df8 100644--- a/internal/service/nav/chunk_agg_router_test.go+++ b/internal/service/nav/chunk_agg_router_test.go@@ -18,6 +18,7 @@ package navimport ("context"+ "errors""math""testing")@@ -60,7 +61,7 @@ func TestAggregateChunks(t *testing.T) {func TestChunkAggRouterRoute(t *testing.T) {ctx := context.Background()rtr := &ChunkAggRouter{- Retrieve: func(_ context.Context, _, _ string, _ string, _ []string, topN int, vecWeight float64) []map[string]any {+ Retrieve: func(_ context.Context, _, _ string, _ string, _ []string, topN int, vecWeight float64) ([]map[string]any, error) {if vecWeight != chunkAggVecWeight {t.Errorf("vecWeight = %v, want %v", vecWeight, chunkAggVecWeight)}@@ -71,7 +72,7 @@ func TestChunkAggRouterRoute(t *testing.T) {{"doc_id": "d1", "similarity": 0.9},{"doc_id": "d1", "similarity": 0.7},{"doc_id": "d2", "similarity": 0.4},- }+ }, nil},Summarize: func(_ context.Context, _, _ string, ids []string) map[string]string {out := map[string]string{}@@ -110,8 +111,8 @@ func TestChunkAggRouterEmpty(t *testing.T) {func TestChunkAggRouterNoRoute(t *testing.T) {ctx := context.Background()rtr := &ChunkAggRouter{- Retrieve: func(_ context.Context, _, _ string, _ string, _ []string, _ int, _ float64) []map[string]any {- return nil+ Retrieve: func(_ context.Context, _, _ string, _ string, _ []string, _ int, _ float64) ([]map[string]any, error) {+ return nil, nil},}routed, err := rtr.Route(ctx, "t", "kb", "q", nil, 5)@@ -125,3 +126,24 @@ func TestChunkAggRouterNoRoute(t *testing.T) {t.Errorf("routed len = %d, want 0", len(routed))}}++// TestChunkAggRouterPropagatesRetrievalError pins the router contract: a+// retrieval FAILURE is an error, not "no document routed". Python's+// _search_layers_nav_chunk_agg returns ok=False/SERVER_ERROR there, reserving+// ok=True/total=0 for an empty aggregation.+func TestChunkAggRouterPropagatesRetrievalError(t *testing.T) {+ ctx := context.Background()+ wantErr := errors.New("ES down")+ rtr := &ChunkAggRouter{+ Retrieve: func(_ context.Context, _, _ string, _ string, _ []string, _ int, _ float64) ([]map[string]any, error) {+ return nil, wantErr+ },+ }+ routed, err := rtr.Route(ctx, "t", "kb", "q", nil, 5)+ if !errors.Is(err, wantErr) {+ t.Fatalf("err = %v, want %v", err, wantErr)+ }+ if routed != nil {+ t.Errorf("routed = %v, want nil alongside the error", routed)+ }+}diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.goindex 7daaf6d0d3..276d523ba9 100644--- a/internal/service/nlp/datasetnav.go+++ b/internal/service/nlp/datasetnav.go@@ -20,6 +20,7 @@ import ("context""encoding/json""fmt"+ "sort""strings""ragflow/internal/engine"@@ -291,25 +292,67 @@ func (s *NavService) Search(ctx context.Context, tenantID, kbID, query string, evec = embeddings[0]}f64 := f32ToF64Slice(vec)- // A scoped read enforces the scope in memory (this read is untyped — it mixes- // nav_doc leaves and nav_cluster rows, so no single query-time key expresses- // both; Python only pushes the scope into the query when type_kwd pins one- // type, dataset_nav.py:1371-1383), so the engine budget must not let- // out-of-scope rows crowd the scope out before the filter runs- // (dataset_nav.py:1384-1387).- scanTop := topK- if len(allowed) > 0 {- scanTop = navScopedScanTopN+ // An unscoped read takes one mixed KNN read: every row is already in scope,+ // so the engine's own topK budget cannot starve the result.+ if len(allowed) == 0 {+ return s.navScan(ctx, tenantID, kbID, navFilter(nil), f64, topK, nil)+ }+ // A scoped read pushes the scope into the query as two type-pinned legs —+ // nav_doc leaves filter on doc_id, nav_cluster rows on doc_ids_kwd — so the+ // ENGINE decides what is in scope, not an in-memory pass over a fixed pool.+ // A fixed pool cannot honor the scope-before-topK contract: a pool of any+ // size N is exhausted by N higher-ranked out-of-scope rows, which silently+ // drops in-scope rows that exist (Python hits the same wall — its mixed read+ // re-checks the scope in memory after a `top_k or 10000` pool, so its typed+ // leg is the only one that actually pushes the scope: dataset_nav.py:1371-1383,+ // 1400, 1444-1447).+ //+ // Two legs are needed because a single mixed condition cannot express both+ // keys at once: `_matches_condition` ANDs across fields, so one condition+ // cannot mean "doc_id IN scope" for leaves and "doc_ids_kwd intersects+ // scope" for clusters (dataset_nav.py:1374-1383 says exactly this).+ scope := navScopeList(allowed)+ leaves, err := s.navScan(ctx, tenantID, kbID, navFilter(map[string]interface{}{+ "type_kwd": []string{nav.TypeNavDoc},+ "doc_id": scope,+ }), f64, topK, allowed)+ if err != nil {+ return nil, err+ }+ clusters, err := s.navScan(ctx, tenantID, kbID, navFilter(map[string]interface{}{+ "type_kwd": []string{nav.TypeNavCluster},+ "doc_ids_kwd": scope,+ }), f64, topK, allowed)+ if err != nil {+ return nil, err+ }+ // Both legs score the same vector with the same cosine metric, so their+ // _score values are directly comparable. Taking each leg's top topK and+ // merging by score yields the global topK of the union: a row ranked below+ // its own leg's cap is already outranked by topK in-scope rows of that leg.+ merged := append(leaves, clusters...)+ sort.SliceStable(merged, func(i, j int) bool { return merged[i].Score > merged[j].Score })+ if len(merged) > topK {+ merged = merged[:topK]}+ return merged, nil+}++// navScan runs one KNN nav read behind filter, maps the rows to NavHits, and+// re-checks the scope in memory the way Python does unconditionally+// (dataset_nav.py:1444). The re-check is belt-and-braces on top of a pushed+// scope filter, and trims a cluster's coverage to the scoped set. allowed may+// be nil for an unscoped read.+func (s *NavService) navScan(ctx context.Context, tenantID, kbID string, filter map[string]interface{}, f64 []float64, limit int, allowed map[string]struct{}) ([]nav.NavHit, error) {chunks, _, err := s.navSearch(ctx, tenantID, kbID,- navFilter(nil),- []string{"type_kwd", "title_kwd", "doc_id", "doc_ids_kwd", "_score"}, 0, scanTop,+ filter,+ []string{"type_kwd", "title_kwd", "doc_id", "doc_ids_kwd", "_score"}, 0, limit,[]interface{}{&types.MatchDenseExpr{VectorColumnName: fmt.Sprintf("q_%d_vec", len(f64)),EmbeddingData: f64,EmbeddingDataType: "float",DistanceType: "cosine",- TopN: scanTop,+ TopN: limit,ExtraOptions: map[string]interface{}{"similarity": 0.0},}})if err != nil {@@ -339,17 +382,21 @@ func (s *NavService) Search(ctx context.Context, tenantID, kbID, query string, eh.DocIDs = navIntersectScope(h.DocIDs, allowed)}hits = append(hits, h)- if len(hits) >= topK {- break- }}return hits, nil}-// navScopedScanTopN is the engine budget for a doc-scoped nav read. It exists so-// the in-memory scope filter runs over a pool wide enough to fill topK with-// in-scope rows even when out-of-scope rows rank higher.-const navScopedScanTopN = 1000+// navScopeList renders a scope set as a sorted list for an engine filter, so the+// term order is deterministic run to run (Python sorts its scope the same way:+// dataset_nav.py:1381-1383).+func navScopeList(allowed map[string]struct{}) []string {+ out := make([]string, 0, len(allowed))+ for d := range allowed {+ out = append(out, d)+ }+ sort.Strings(out)+ return out+}// navScopeSet normalizes a caller's doc scope the way search_dataset_nav does// (dataset_nav.py:1364): trimmed, non-empty, deduped. A nil result means "nodiff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.goindex 05f350b27c..214e80832c 100644--- a/internal/service/nlp/datasetnav_test.go+++ b/internal/service/nlp/datasetnav_test.go@@ -170,33 +170,61 @@ func (m *memNavEngine) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []sreturn nil}+// matchNavRow mirrors dataset_nav._matches_condition (dataset_nav.py:598-608):+// every field must have one of the row's values equal to one of the condition's+// values. A list condition therefore means intersection when the row's field is+// an array (the scope push for nav_cluster's doc_ids_kwd) and membership when+// the field is a scalar. The real engines give a list filter the same semantics+// (elasticsearch/chunk.go terms, serenedb list_contains, oceanbase+// ARRAY_CONTAINS).func matchNavRow(row map[string]interface{}, cond map[string]interface{}) bool {for k, v := range cond {rv, ok := row[k]if !ok {return false}- switch want := v.(type) {- case []string:- ok = false- for _, w := range want {- if rv == w {- ok = true- break- }- }- if !ok {- return false- }- default:- if rv != v {- return false- }+ if !navValueMatches(rv, v) {+ return false}}return true}+// navValueMatches compares one field the way `_matches_condition` does+// (`str(value) == str(item)`), with list-vs-list compared as an intersection.+func navValueMatches(actual, expected interface{}) bool {+ actuals, actualIsList := navValueList(actual)+ expecteds, expectedIsList := navValueList(expected)+ if !actualIsList && !expectedIsList {+ return fmt.Sprint(actual) == fmt.Sprint(expected)+ }+ for _, a := range actuals {+ for _, e := range expecteds {+ if a == e {+ return true+ }+ }+ }+ return false+}++// navValueList normalizes a row or condition value to a list of comparable+// scalars, reporting whether it was already a list.+func navValueList(v interface{}) ([]interface{}, bool) {+ switch t := v.(type) {+ case []string:+ out := make([]interface{}, 0, len(t))+ for _, x := range t {+ out = append(out, x)+ }+ return out, true+ case []interface{}:+ return t, true+ default:+ return []interface{}{v}, false+ }+}+func scoreNavOf(r map[string]interface{}) float64 {switch s := r["_score"].(type) {case float64:@@ -458,7 +486,7 @@ func TestNavService_Search_DocScope(t *testing.T) {}// Three out-of-scope leaves outrank the single in-scope leaf (cosine 1.0 vs// 0.5), so a scoped read truncated to one row only fills its cap if the- // scope is applied over a wide-enough pool.+ // scope is decided by the engine rather than after the truncation.rows := []map[string]interface{}{row(nav.TypeNavDoc, "dx1", "out 1", nil, 1),row(nav.TypeNavDoc, "dx2", "out 2", nil, 1),@@ -508,9 +536,9 @@ func TestNavService_Search_DocScope(t *testing.T) {}}- // The scope is applied to the pool, not after the engine truncation: the- // top-ranked rows here are all out of scope, yet the in-scope leaf still- // comes back as the single hit.+ // The scope is pushed into the query, not applied after the engine+ // truncation: the top-ranked rows here are all out of scope, yet the+ // in-scope leaf still comes back as the single hit.one, err := ns.Search(t.Context(), "t1", "kb1", "q", q, []string{"d1"}, 1)if err != nil {t.Fatal(err)@@ -520,6 +548,94 @@ func TestNavService_Search_DocScope(t *testing.T) {}}+// TestNavService_Search_DocScopeBeyondAnyPoolSize pins the engine-budget half of+// the scope contract: a scoped read must not come back empty just because more+// out-of-scope rows rank above every in-scope row than the engine budget can+// hold. The scope is pushed into the query as two type-pinned legs, so no pool+// size can hide an in-scope row — whereas the fixed 1000-row pool this read used+// to scan is exhausted by the 1200 out-of-scope rows below.+func TestNavService_Search_DocScopeBeyondAnyPoolSize(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)++ const dim = 4+ vec := func(w float64) []float64 {+ v := make([]float64, dim)+ v[0] = w+ return v+ }+ // 1200 out-of-scope leaves, all ranked above the in-scope rows (cosine 1.0+ // vs 0.5 / 0.4).+ rows := make([]map[string]interface{}, 0, 1202)+ for i := 0; i < 1200; i++ {+ rows = append(rows, map[string]interface{}{+ "compile_kwd": navCompileKwd,+ "type_kwd": nav.TypeNavDoc,+ "doc_id": fmt.Sprintf("dx%d", i),+ "title_kwd": fmt.Sprintf("out %d", i),+ "q_4_vec": vec(1),+ })+ }+ rows = append(rows,+ map[string]interface{}{+ "compile_kwd": navCompileKwd,+ "type_kwd": nav.TypeNavDoc,+ "doc_id": "d1",+ "title_kwd": "in scope",+ "q_4_vec": vec(0.5),+ },+ map[string]interface{}{+ "compile_kwd": navCompileKwd,+ "type_kwd": nav.TypeNavCluster,+ "doc_id": "kb1",+ "title_kwd": "covering d1+d2",+ "doc_ids_kwd": []string{"d1", "d2"},+ "q_4_vec": vec(0.4),+ },+ )+ if _, err := eng.InsertChunks(t.Context(), rows, "", "kb1"); err != nil {+ t.Fatal(err)+ }+ q := make([]float32, dim)+ q[0] = 1++ // Sanity: unscoped, the in-scope rows really do rank last, so a read that+ // mixes the scope in only after the engine truncation comes back empty.+ unscoped, err := ns.Search(t.Context(), "t1", "kb1", "q", q, nil, 2)+ if err != nil {+ t.Fatal(err)+ }+ if len(unscoped) != 2 {+ t.Fatalf("unscoped hits = %d, want 2", len(unscoped))+ }+ for _, h := range unscoped {+ if h.DocID == "d1" || h.DocID == "kb1" {+ t.Fatalf("unscoped top 2 = %+v, want the two highest-ranked out-of-scope leaves", unscoped)+ }+ }++ scoped, err := ns.Search(t.Context(), "t1", "kb1", "q", q, []string{"d1"}, 2)+ if err != nil {+ t.Fatal(err)+ }+ if len(scoped) != 2 || scoped[0].DocID != "d1" || scoped[1].DocID != "kb1" {+ t.Fatalf("scoped hits = %+v, want [d1 (0.5), kb1 cluster (0.4)] despite 1200 higher-ranked out-of-scope rows", scoped)+ }+ if len(scoped[1].DocIDs) != 1 || scoped[1].DocIDs[0] != "d1" {+ t.Errorf("cluster coverage = %v, want [d1] (trimmed to the scope)", scoped[1].DocIDs)+ }++ // A scope with no in-scope row at all stays empty rather than falling back+ // to the out-of-scope pool.+ none, err := ns.Search(t.Context(), "t1", "kb1", "q", q, []string{"d-absent"}, 2)+ if err != nil {+ t.Fatal(err)+ }+ if len(none) != 0 {+ t.Fatalf("scoped hits for an absent doc = %+v, want none", none)+ }+}+// TestNavService_Acceptance4_ListChildren asserts acceptance #4: ListChildren// returns only the rows whose parent_kwd=name. Two docs with identical stub// vectors merge into one root cluster: the first becomes the cluster itself,diff --git a/internal/tokenizer/failfast_test.go b/internal/tokenizer/failfast_test.goindex 4567c66487..ac386299e0 100644--- a/internal/tokenizer/failfast_test.go+++ b/internal/tokenizer/failfast_test.go@@ -29,12 +29,13 @@ import (// guards — that a PRESENT table yields a working encoder (NumTokensFromString > 0)// rather than a silent 0.//-// Background: NumTokensFromString swallows the loader error and returns 0, so a-// Go image that forgot to bake cl100k_base.tiktoken silently zeroed every token-// count while content_ltks (a separate C++ tokenizer) kept working. The fix is-// to fail fast at startup via InitCL100KEncoder; this test must exercise BOTH-// branches, otherwise it only re-asserts "missing → error" and leaves the-// "present → counts" behaviour — the thing that regressed — uncovered.+// Background: a Go image that forgot to bake cl100k_base.tiktoken silently zeroed+// every token count while content_ltks (a separate C++ tokenizer) kept working.+// TWO guards now cover it: InitCL100KEncoder fails fast at startup, and+// NumTokensFromString / TrimContentToTokenLimit PANIC on a missing table+// (mirroring Python's num_tokens_from_string / trim_content, which resolve the+// encoder outside their try). This test exercises BOTH branches — including the+// panic — because the "present → counts" behaviour is what regressed.//// Both branches reset the encoder cache and scope bpeSearchRoots to a temp dir,// so neither depends on the host (no leaked tiktoken cache in an ancestor, no@@ -58,6 +59,18 @@ func TestInitCL100KEncoder_FailFast(t *testing.T) {if !strings.Contains(err.Error(), "cl100k") {t.Fatalf("InitCL100KEncoder error does not mention cl100k: %v", err)}++ // Python raises here; returning 0 (or byte-trimming) would fail OPEN.+ mustPanic := func(what string, fn func()) {+ defer func() {+ if recover() == nil {+ t.Fatalf("%s: expected a panic, got none", what)+ }+ }()+ fn()+ }+ mustPanic("NumTokensFromString with no table", func() { NumTokensFromString("hello world") })+ mustPanic("TrimContentToTokenLimit with no table", func() { TrimContentToTokenLimit("hello world", 1) })})// present: copy a real table into an isolated dir (via TIKTOKEN_CACHE_DIR sodiff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.goindex 6e6fc51b52..b85a9963c4 100644--- a/internal/tokenizer/tokenizer.go+++ b/internal/tokenizer/tokenizer.go@@ -24,7 +24,6 @@ import ("sync""sync/atomic""time"- "unicode/utf8""github.com/pkoukk/tiktoken-go""go.uber.org/zap"@@ -596,20 +595,26 @@ func InitCL100KEncoder() error {}// NumTokensFromString returns the number of tokens in s using the cl100k_base-// BPE encoding. Mirrors Python's num_tokens_from_string (common/token_utils.py):-// returns 0 on encoder error.+// BPE encoding.//-// Callers must ensure InitCL100KEncoder has succeeded at startup; otherwise a-// missing table makes this return 0 for every non-empty string. The startup-// check is the fail-fast guard — this cheap hot-path function stays silent by-// design so it can be called per-token without error plumbing.+// A missing BPE table PANICS instead of returning 0. Python's+// num_tokens_from_string (common/token_utils.py) resolves its encoder OUTSIDE the+// try, so an unavailable table raises there — only an encode() failure is folded+// into 0. Returning 0 here would fail OPEN (every budget would "fit"), which is+// the exact silent-corruption mode that made a Go image missing+// cl100k_base.tiktoken zero every token count. InitCL100KEncoder still runs at+// startup (cmd/ragflow_server.go) so a bad deployment dies with a clear message,+// and this function now stays loud too in case a caller bypasses that guard.func NumTokensFromString(s string) int {if s == "" {return 0}enc, err := getCL100KEncoder()if err != nil {- return 0+ panic(fmt.Sprintf("tokenizer.NumTokensFromString: cl100k_base BPE table unavailable: %v", err))+ }+ if enc == nil {+ panic("tokenizer.NumTokensFromString: cl100k_base encoder is nil")}return len(enc.Encode(s, nil, nil))}@@ -618,24 +623,20 @@ func NumTokensFromString(s string) int {// cl100k_base encoder. Mirrors Python's trim_content helper in// rag/prompts/generator.py: encoder.decode(encoder.encode(content)[:limit]).// Returns the original string if it already fits.+//+// Like Python's trim_content, an unavailable encoder PANICS: the previous+// byte-length fallback silently produced a differently-truncated string instead+// of surfacing the missing table. See NumTokensFromString.func TrimContentToTokenLimit(s string, limit int) string {if limit < 0 {limit = 0}enc, err := getCL100KEncoder()if err != nil {- // Fail closed: fall back to byte-length trimming with UTF-8 safety.- if limit <= 0 {- return ""- }- b := []byte(s)- if len(b) <= limit {- return s- }- for limit > 0 && !utf8.Valid(b[:limit]) {- limit--- }- return string(b[:limit])+ panic(fmt.Sprintf("tokenizer.TrimContentToTokenLimit: cl100k_base BPE table unavailable: %v", err))+ }+ if enc == nil {+ panic("tokenizer.TrimContentToTokenLimit: cl100k_base encoder is nil")}tokens := enc.Encode(s, nil, nil)if len(tokens) <= limit {diff --git a/rag/advanced_rag/harness/memory.py b/rag/advanced_rag/harness/memory.pyindex 46ded2be86..04256d1972 100644--- a/rag/advanced_rag/harness/memory.py+++ b/rag/advanced_rag/harness/memory.py@@ -179,6 +179,12 @@ def grep(tools, terms, limit: int = _GREP_MAX_CHUNKS) -> list[dict]:# Short chunk: keep whole (its answer may live anywhere in it).if _match(text):hits.append({"content": text, "doc_id": c.get("doc_id"), "chunk_id": c.get("chunk_id")})+ # This branch `continue`s past the shared cap at the bottom of the+ # loop, so enforce the limit here too — otherwise a memory store+ # full of matching short chunks comes back whole instead of at+ # most ``limit`` entries.+ if len(hits) >= limit:+ breakcontinuesents = _split_sentences(text)kept = []@@ -338,10 +344,7 @@ def _term_hits(text: str, terms: list[str]) -> int:return 0hits = 0for t in terms:- if _is_cjk(t):- if t in text:- hits += 1- elif t.isdigit():+ if _is_cjk(t) or t.isdigit():if t in text:hits += 1else:diff --git a/rag/advanced_rag/harness/orchestrator/sufficient_context.py b/rag/advanced_rag/harness/orchestrator/sufficient_context.pyindex 5d68bcb258..6aaf3dc105 100644--- a/rag/advanced_rag/harness/orchestrator/sufficient_context.py+++ b/rag/advanced_rag/harness/orchestrator/sufficient_context.py@@ -81,6 +81,27 @@ def _clamp(value, lo: float = 0.0, hi: float = 1.0) -> float:return 1.0+_TRUE_STRINGS = frozenset({"true", "1", "yes", "y", "on"})+++def _coerce_bool(value) -> bool:+ """Read an LLM-reported boolean, tolerating string drift.++ ``bool()`` alone treats the literal strings ``"false"`` and ``"0"`` as True+ (they are non-empty), which silently INVERTS a verdict — marking insufficient+ context sufficient, or an ungrounded claim grounded. Read the recognised+ spellings by meaning instead; empty/unrecognised values fail CLOSED (False),+ matching this module's convention that an unusable verdict is INSUFFICIENT.+ """+ if isinstance(value, bool):+ return value+ if isinstance(value, str):+ return value.strip().lower() in _TRUE_STRINGS+ if isinstance(value, (int, float)):+ return value != 0+ return False++def _coerce_dict(result) -> dict | None:"""Coerce a ``gen_json`` response into a dict, tolerating model format drift.@@ -175,10 +196,8 @@ def _render_claim_context(claims, question: str = "", kbinfos: dict | None = Nonif not rpt:continueblock = f"Claim {cid} (draft):\n{rpt}"- used += len(block) + 2# Evidence anchor: first line of each cited snippet (guards the draft).- # Table chunks contribute their FULL text (see _bounded_excerpt), so- # account the actual joined length, not the 300-char estimate.+ # Table chunks contribute their FULL text (see _bounded_excerpt).if eids:anchors: list[str] = []for eid in eids:@@ -194,10 +213,18 @@ def _render_claim_context(claims, question: str = "", kbinfos: dict | None = Nonif len(anchors) >= 3:breakif anchors:- anchor_text = " | ".join(anchors)- block += "\n Evidence: " + anchor_text- used += len(anchor_text) + 4+ block += "\n Evidence: " + " | ".join(anchors)+ # Apply the budget to the COMPLETE block before appending it, trimming it to+ # what remains: accounting the block only AFTER appending let a single+ # oversized claim blow straight past _SCA_CLAIMS_CONTEXT_MAX (the cap the+ # comment claims bounds the whole rendered context).+ remaining = _SCA_CLAIMS_CONTEXT_MAX - used+ if remaining <= 0:+ break+ if len(block) > remaining:+ block = block[:remaining]blocks.append(block)+ used += len(block) + 2 # the block plus the "\n\n" join separatorif used >= _SCA_CLAIMS_CONTEXT_MAX:breakreturn "\n\n".join(blocks) if blocks else "(no claim drafts)"@@ -306,7 +333,7 @@ async def sufficient_context_agent(elif m:missing_info.append({"what": str(m).strip(), "search_hint": ""})claims_out[cid] = {- "grounded": bool(item.get("grounded")),+ "grounded": _coerce_bool(item.get("grounded")),"ungrounded": [a for a in ungrounded if a],"missing_information": missing_info,}@@ -325,14 +352,14 @@ async def sufficient_context_agent(continuesq_out: dict = {"sub_query": sq_text,- "satisfied": bool(sq.get("satisfied")),+ "satisfied": _coerce_bool(sq.get("satisfied")),}if not sq_out["satisfied"]:sq_out["missing_fact"] = str(sq.get("missing_fact") or "").strip()sq_out["search_hint"] = str(sq.get("search_hint") or "").strip()sub_queries.append(sq_out)- is_sufficient = bool(result.get("is_sufficient"))+ is_sufficient = _coerce_bool(result.get("is_sufficient"))# Failsafe: when the SCA judges the context insufficient but returned an EMPTY# claims array (a known degradation on very long prompts), we must still give# the orchestrator something to re-search. Otherwise it abandons with@@ -396,7 +423,7 @@ def to_boost(sca: dict, verdict, fallback_followups: list | None = None) -> dictif missing:feedback = "missing: " + "; ".join(missing[:_FEEDBACK_MAX])return {- "is_sufficient": bool(sca.get("is_sufficient")),+ "is_sufficient": _coerce_bool(sca.get("is_sufficient")),"confidence": _clamp(sca.get("confidence")),"missing": missing,"contradictions": contradictions,diff --git a/rag/prompts/generator.py b/rag/prompts/generator.pyindex 383e379b49..e88ea4edde 100644--- a/rag/prompts/generator.py+++ b/rag/prompts/generator.py@@ -19,15 +19,15 @@ import jsonimport loggingimport refrom copy import deepcopy-from typing import Tuple-from jinja2.sandbox import SandboxedEnvironment+import json_repair+from jinja2.sandbox import SandboxedEnvironment+from common.constants import TAG_FLDfrom common.misc_utils import hash_str2int+from common.token_utils import get_encoder, num_tokens_from_stringfrom rag.nlp import rag_tokenizerfrom rag.prompts.template import load_prompt-from common.constants import TAG_FLD-from common.token_utils import get_encoder, num_tokens_from_stringSTOP_TOKEN = "<|STOP|>"COMPLETE_TASK = "complete_task"@@ -137,21 +137,15 @@ def message_fit_in(msg, max_length=4000):return count(), msg-def kb_prompt(kbinfos, max_tokens, hash_id=False):- chunks = kbinfos["chunks"]- knowledges = [get_value(ck, "content", "content_with_weight") for ck in chunks]- kwlg_len = len(knowledges)- used_token_count = 0- selected_chunks = []- for ck, c in zip(chunks, knowledges):- if not c:- continue- chunk_tokens = num_tokens_from_string(c)- if max_tokens * 0.97 < used_token_count + chunk_tokens:- logging.warning(f"Not all the retrieval into prompt: {len(selected_chunks)}/{kwlg_len}")- break- used_token_count += chunk_tokens- selected_chunks.append(ck)+def _kb_block(ck, index: int, hash_id: bool = False) -> str | None:+ """Render one knowledge block (id / title / url / metadata / content).++ Returns None when the chunk carries no content, which kb_prompt treats as+ "skip" (mirrors `if not c: continue`).+ """+ content = get_value(ck, "content", "content_with_weight")+ if not content:+ return Nonedef draw_node(k, line):if line is not None and not isinstance(line, str):@@ -160,19 +154,35 @@ def kb_prompt(kbinfos, max_tokens, hash_id=False):return ""return f"\n├── {k}: " + re.sub(r"\n+", " ", line, flags=re.DOTALL)- knowledges = []- for i, ck in enumerate(selected_chunks):- cnt = "\nID: {}".format(i if not hash_id else hash_str2int(get_value(ck, "id", "chunk_id"), 500))- cnt += draw_node("Title", get_value(ck, "docnm_kwd", "document_name"))- cnt += draw_node("URL", ck.get("url", ""))- meta = ck.get("document_metadata") or {}- for k, v in meta.items():- cnt += draw_node(k, v)- cnt += "\n└── Content:\n"- cnt += get_value(ck, "content", "content_with_weight")- knowledges.append(cnt)+ cnt = "\nID: {}".format(index if not hash_id else hash_str2int(get_value(ck, "id", "chunk_id"), 500))+ cnt += draw_node("Title", get_value(ck, "docnm_kwd", "document_name"))+ cnt += draw_node("URL", ck.get("url", ""))+ meta = ck.get("document_metadata") or {}+ for k, v in meta.items():+ cnt += draw_node(k, v)+ cnt += "\n└── Content:\n"+ cnt += content+ return cnt- return knowledges++def kb_prompt(kbinfos, max_tokens, hash_id=False):+ chunks = kbinfos["chunks"]+ used_token_count = 0+ out: list[str] = []+ for ck in chunks:+ # Budget the COMPLETE block: the title / url / metadata decoration was+ # previously invisible to the budget, so the rendered prompt could exceed+ # the max_tokens it claims to enforce.+ block = _kb_block(ck, len(out) + 1, hash_id)+ if block is None:+ continue+ block_tokens = num_tokens_from_string(block)+ if max_tokens * 0.97 < used_token_count + block_tokens:+ logging.warning(f"Not all the retrieval into prompt: {len(out)}/{len(chunks)}")+ break+ used_token_count += block_tokens+ out.append(block)+ return outdef memory_prompt(message_list, max_tokens):@@ -254,9 +264,9 @@ async def question_proposal(chat_mdl, content, topn=3):async def full_question(tenant_id=None, llm_id=None, messages=[], language=None, chat_mdl=None):- from common.constants import LLMType- from api.db.services.llm_service import LLMBundlefrom api.db.joint_services.tenant_model_service import resolve_model_config, resolve_model_type+ from api.db.services.llm_service import LLMBundle+ from common.constants import LLMTypeif not chat_mdl:model_types = resolve_model_type(tenant_id, llm_id)@@ -290,13 +300,13 @@ async def full_question(tenant_id=None, llm_id=None, messages=[], language=None,async def cross_languages(tenant_id, llm_id, query, languages=[]):- from common.constants import LLMType- from api.db.services.llm_service import LLMBundlefrom api.db.joint_services.tenant_model_service import (get_tenant_default_model_by_type,resolve_model_config,resolve_model_type,)+ from api.db.services.llm_service import LLMBundle+ from common.constants import LLMTypeif llm_id and "vision" in resolve_model_type(tenant_id, llm_id):chat_model_config = resolve_model_config(tenant_id, LLMType.VISION, llm_id)@@ -460,7 +470,7 @@ async def next_step_async(chat_mdl, history: list, tools_description: list[dict]return json_str, tk_cnt-async def reflect_async(chat_mdl, history: list[dict], tool_call_res: list[Tuple], user_defined_prompts: dict = {}):+async def reflect_async(chat_mdl, history: list[dict], tool_call_res: list[tuple], user_defined_prompts: dict = {}):tool_calls = [{"name": p[0], "result": p[1]} for p in tool_call_res]goal = history[1]["content"]template = PROMPT_JINJA_ENV.from_string(user_defined_prompts.get("reflection", REFLECT))@@ -473,13 +483,13 @@ async def reflect_async(chat_mdl, history: list[dict], tool_call_res: list[Tuple_, msg = message_fit_in(hist, chat_mdl.max_length)ans = await chat_mdl.async_chat(msg[0]["content"], msg[1:])ans = re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)- return """+ return f"""**Observation**-{}+{json.dumps(tool_calls, ensure_ascii=False, indent=2)}**Reflection**-{}- """.format(json.dumps(tool_calls, ensure_ascii=False, indent=2), ans)+{ans}+ """def form_message(system_prompt, user_prompt):@@ -835,7 +845,7 @@ def split_chunks(chunks, max_length: int):async def run_toc_from_text(chunks, chat_mdl, callback=None):input_budget = int(chat_mdl.max_length * INPUT_UTILIZATION) - num_tokens_from_string(TOC_FROM_TEXT_USER + TOC_FROM_TEXT_SYSTEM)- input_budget = 1024 if input_budget > 1024 else input_budget+ input_budget = min(input_budget, 1024)chunk_sections = split_chunks(chunks, input_budget)titles = []@@ -945,7 +955,7 @@ async def relevant_chunks_with_toc(query: str, toc: list[dict], chat_mdl, topn:if id not in id2score:id2score[id] = []id2score[id].append(sc["score"] / 5.0)- for id in id2score.keys():+ for id in id2score:id2score[id] = np.mean(id2score[id])return [(id, sc) for id, sc in list(id2score.items()) if sc >= 0.3][:topn]except Exception as e: